drivers/gpu/rogue_m/services/server/devices/rgx/rgxfwutils.c

   1  /*************************************************************************/ /*!
   2 @File
   3 @Title          Rogue firmware utility routines
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Rogue firmware utility routines
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43
  44 #include <stddef.h>
  45
  46 #include "lists.h"
  47
  48 #include "rgxdefs_km.h"
  49 #include "rgx_fwif_km.h"
  50 #include "pdump_km.h"
  51 #include "osfunc.h"
  52 #include "allocmem.h"
  53 #include "devicemem.h"
  54 #include "devicemem_pdump.h"
  55 #include "devicemem_server.h"
  56 #include "pvr_debug.h"
  57 #include "rgxfwutils.h"
  58 #include "rgx_fwif.h"
  59 #include "rgx_fwif_alignchecks_km.h"
  60 #include "rgx_fwif_resetframework.h"
  61 #include "rgx_pdump_panics.h"
  62 #include "rgxheapconfig.h"
  63 #include "pvrsrv.h"
  64 #include "rgxdebug.h"
  65 #include "rgxhwperf.h"
  66 #include "rgxccb.h"
  67 #include "rgxcompute.h"
  68 #include "rgxtransfer.h"
  69 #if defined(RGX_FEATURE_RAY_TRACING)
  70 #include "rgxray.h"
  71 #endif
  72 #if defined(SUPPORT_DISPLAY_CLASS)
  73 #include "dc_server.h"
  74 #endif
  75 #include "rgxmem.h"
  76 #include "rgxta3d.h"
  77 #include "rgxutils.h"
  78 #include "sync_internal.h"
  79 #include "tlstream.h"
  80 #include "devicemem_server_utils.h"
  81
  82 #if defined(TDMETACODE)
  83 #include "physmem_osmem.h"
  84 #endif
  85
  86 #ifdef __linux__
  87 #include <linux/kernel.h>       // sprintf
  88 #include <linux/string.h>       // strncpy, strlen
  89 #include "trace_events.h"
  90 #else
  91 #include <stdio.h>
  92 #endif
  93
  94 #include "process_stats.h"
  95 /* Kernel CCB length */
  96 #define RGXFWIF_KCCB_TA_NUMCMDS_LOG2    (6)
  97 #define RGXFWIF_KCCB_3D_NUMCMDS_LOG2    (6)
  98 #define RGXFWIF_KCCB_2D_NUMCMDS_LOG2    (6)
  99 #define RGXFWIF_KCCB_CDM_NUMCMDS_LOG2   (6)
 100 #define RGXFWIF_KCCB_GP_NUMCMDS_LOG2    (6)
 101 #define RGXFWIF_KCCB_RTU_NUMCMDS_LOG2   (6)
 102 #define RGXFWIF_KCCB_SHG_NUMCMDS_LOG2   (6)
 103
 104 /* Firmware CCB length */
 105 #define RGXFWIF_FWCCB_TA_NUMCMDS_LOG2   (4)
 106 #define RGXFWIF_FWCCB_3D_NUMCMDS_LOG2   (4)
 107 #define RGXFWIF_FWCCB_2D_NUMCMDS_LOG2   (4)
 108 #define RGXFWIF_FWCCB_CDM_NUMCMDS_LOG2  (4)
 109 #define RGXFWIF_FWCCB_GP_NUMCMDS_LOG2   (4)
 110 #define RGXFWIF_FWCCB_RTU_NUMCMDS_LOG2  (4)
 111 #define RGXFWIF_FWCCB_SHG_NUMCMDS_LOG2  (4)
 112
 113 #if defined(RGX_FEATURE_SLC_VIVT)
 114 static PVRSRV_ERROR _AllocateSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo, RGXFWIF_INIT* psRGXFWInit)
 115 {
 116         PVRSRV_ERROR eError;
 117         DEVMEM_MEMDESC** ppsSLC3FenceMemDesc = &psDevInfo->psSLC3FenceMemDesc;
 118
 119         PVR_DPF_ENTERED;
 120
 121         eError = DevmemAllocate(psDevInfo->psFirmwareHeap,
 122                                                         1,
 123                                                         ROGUE_CACHE_LINE_SIZE,
 124                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 125                             PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 126                                                         PVRSRV_MEMALLOCFLAG_UNCACHED,
 127                                                         "SLC3 Fence WA",
 128                                                         ppsSLC3FenceMemDesc);
 129         if (eError != PVRSRV_OK)
 130         {
 131                 PVR_DPF_RETURN_RC(eError);
 132         }
 133
 134         /*
 135                 We need to map it so the heap for this allocation
 136                 is set
 137         */
 138         eError = DevmemMapToDevice(*ppsSLC3FenceMemDesc,
 139                                                            psDevInfo->psFirmwareHeap,
 140                                                            &psRGXFWInit->sSLC3FenceDevVAddr);
 141         if (eError != PVRSRV_OK)
 142         {
 143                 DevmemFwFree(*ppsSLC3FenceMemDesc);
 144         }
 145
 146         PVR_DPF_RETURN_RC1(eError, *ppsSLC3FenceMemDesc);
 147 }
 148
 149 static IMG_VOID _FreeSLC3Fence(PVRSRV_RGXDEV_INFO* psDevInfo)
 150 {
 151         DEVMEM_MEMDESC* psSLC3FenceMemDesc = psDevInfo->psSLC3FenceMemDesc;
 152
 153         if (psSLC3FenceMemDesc)
 154         {
 155                 DevmemReleaseDevVirtAddr(psSLC3FenceMemDesc);
 156                 DevmemFree(psSLC3FenceMemDesc);
 157         }
 158 }
 159 #endif
 160
 161 static IMG_VOID __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
 162 {
 163         /* ensure memory is flushed before kicking MTS */
 164         OSWriteMemoryBarrier();
 165
 166         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
 167
 168         /* ensure the MTS kick goes through before continuing */
 169         OSMemoryBarrier();
 170 }
 171
 172
 173 /*!
 174 *******************************************************************************
 175  @Function              RGXFWSetupSignatureChecks
 176  @Description
 177  @Input                 psDevInfo
 178
 179  @Return                PVRSRV_ERROR
 180 ******************************************************************************/
 181 static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
 182                                               DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
 183                                               IMG_UINT32          ui32SigChecksBufSize,
 184                                               RGXFWIF_SIGBUF_CTL* psSigBufCtl,
 185                                               const IMG_CHAR*     pszBufferName)
 186 {
 187         PVRSRV_ERROR    eError;
 188         DEVMEM_FLAGS_T  uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 189                                                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 190                                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 191                                                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 192                                                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 193                                                                           PVRSRV_MEMALLOCFLAG_UNCACHED |
 194                                                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 195
 196         /* Allocate memory for the checks */
 197         PDUMPCOMMENT("Allocate memory for %s signature checks", pszBufferName);
 198         eError = DevmemFwAllocate(psDevInfo,
 199                                                         ui32SigChecksBufSize,
 200                                                         uiMemAllocFlags,
 201                                                         "SignatureChecks",
 202                                                         ppsSigChecksMemDesc);
 203         if (eError != PVRSRV_OK)
 204         {
 205                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for signature checks (%u)",
 206                                         ui32SigChecksBufSize,
 207                                         eError));
 208                 return eError;
 209         }
 210
 211         /* Prepare the pointer for the fw to access that memory */
 212         RGXSetFirmwareAddress(&psSigBufCtl->psBuffer,
 213                                                   *ppsSigChecksMemDesc,
 214                                                   0, RFW_FWADDR_NOREF_FLAG);
 215
 216         DevmemPDumpLoadMem(     *ppsSigChecksMemDesc,
 217                                                 0,
 218                                                 ui32SigChecksBufSize,
 219                                                 PDUMP_FLAGS_CONTINUOUS);
 220
 221         psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
 222
 223         return PVRSRV_OK;
 224 }
 225
 226 #if defined(RGXFW_ALIGNCHECKS)
 227 /*!
 228 *******************************************************************************
 229  @Function              RGXFWSetupAlignChecks
 230  @Description
 231  @Input                 psDevInfo
 232
 233  @Return                PVRSRV_ERROR
 234 ******************************************************************************/
 235 static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
 236                                                                 RGXFWIF_DEV_VIRTADDR    *psAlignChecksDevFW,
 237                                                                 IMG_UINT32                              *pui32RGXFWAlignChecks,
 238                                                                 IMG_UINT32                              ui32RGXFWAlignChecksSize)
 239 {
 240         IMG_UINT32              aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
 241         IMG_UINT32              ui32RGXFWAlingChecksTotal = sizeof(aui32RGXFWAlignChecksKM) + ui32RGXFWAlignChecksSize;
 242         IMG_UINT32*             paui32AlignChecks;
 243         PVRSRV_ERROR    eError;
 244
 245         /* Allocate memory for the checks */
 246         PDUMPCOMMENT("Allocate memory for alignment checks");
 247         eError = DevmemFwAllocate(psDevInfo,
 248                                                         ui32RGXFWAlingChecksTotal,
 249                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 250                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 251                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 252                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 253                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | PVRSRV_MEMALLOCFLAG_UNCACHED,
 254                                                         "AlignmentChecks",
 255                                                         &psDevInfo->psRGXFWAlignChecksMemDesc);
 256         if (eError != PVRSRV_OK)
 257         {
 258                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for alignment checks (%u)",
 259                                         ui32RGXFWAlingChecksTotal,
 260                                         eError));
 261                 goto failAlloc;
 262         }
 263
 264         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
 265                                                                         (IMG_VOID **)&paui32AlignChecks);
 266         if (eError != PVRSRV_OK)
 267         {
 268                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel addr for alignment checks (%u)",
 269                                         eError));
 270                 goto failAqCpuAddr;
 271         }
 272
 273         /* Copy the values */
 274         OSMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0], sizeof(aui32RGXFWAlignChecksKM));
 275         paui32AlignChecks += sizeof(aui32RGXFWAlignChecksKM)/sizeof(IMG_UINT32);
 276
 277         OSMemCopy(paui32AlignChecks, pui32RGXFWAlignChecks, ui32RGXFWAlignChecksSize);
 278
 279         DevmemPDumpLoadMem(     psDevInfo->psRGXFWAlignChecksMemDesc,
 280                                                 0,
 281                                                 ui32RGXFWAlingChecksTotal,
 282                                                 PDUMP_FLAGS_CONTINUOUS);
 283
 284         /* Prepare the pointer for the fw to access that memory */
 285         RGXSetFirmwareAddress(psAlignChecksDevFW,
 286                                                   psDevInfo->psRGXFWAlignChecksMemDesc,
 287                                                   0, RFW_FWADDR_NOREF_FLAG);
 288
 289         return PVRSRV_OK;
 290
 291
 292
 293
 294 failAqCpuAddr:
 295         DevmemFwFree(psDevInfo->psRGXFWAlignChecksMemDesc);
 296 failAlloc:
 297
 298         PVR_ASSERT(eError != PVRSRV_OK);
 299         return eError;
 300 }
 301
 302 static IMG_VOID RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
 303 {
 304         if (psDevInfo->psRGXFWAlignChecksMemDesc != IMG_NULL)
 305         {
 306                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
 307                 DevmemFwFree(psDevInfo->psRGXFWAlignChecksMemDesc);
 308                 psDevInfo->psRGXFWAlignChecksMemDesc = IMG_NULL;
 309         }
 310 }
 311 #endif
 312
 313
 314 IMG_VOID RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR     *ppDest,
 315                                                            DEVMEM_MEMDESC               *psSrc,
 316                                                            IMG_UINT32                   uiExtraOffset,
 317                                                            IMG_UINT32                   ui32Flags)
 318 {
 319         PVRSRV_ERROR            eError;
 320         IMG_DEV_VIRTADDR        psDevVirtAddr;
 321         IMG_UINT64                      ui64Offset;
 322         IMG_BOOL            bCachedInMETA;
 323         DEVMEM_FLAGS_T      uiDevFlags;
 324
 325         eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
 326         PVR_ASSERT(eError == PVRSRV_OK);
 327
 328         /* Convert to an address in META memmap */
 329         ui64Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_HEAP_BASE;
 330
 331         /* The biggest offset for the Shared region that can be addressed */
 332         PVR_ASSERT(ui64Offset < 3*RGXFW_SEGMMU_DMAP_SIZE);
 333
 334         /* Check in the devmem flags whether this memory is cached/uncached */
 335         DevmemGetFlags(psSrc, &uiDevFlags);
 336
 337         /* Honour the META cache flags */
 338         bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) & uiDevFlags) != 0;
 339
 340 #if defined(HW_ERN_45914)
 341         /* We only cache in META if it's also cached in the SLC */
 342         {
 343                 IMG_BOOL bCachedInSLC = (DevmemDeviceCacheMode(uiDevFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHED);
 344
 345                 bCachedInMETA = bCachedInMETA && bCachedInSLC;
 346         }
 347 #endif
 348
 349         if (bCachedInMETA)
 350         {
 351                 ppDest->ui32Addr = ((IMG_UINT32) ui64Offset) | RGXFW_BOOTLDR_META_ADDR;
 352         }
 353         else
 354         {
 355                 ppDest->ui32Addr = ((IMG_UINT32) ui64Offset) | RGXFW_SEGMMU_DMAP_ADDR_START;
 356         }
 357
 358         if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
 359         {
 360                 DevmemReleaseDevVirtAddr(psSrc);
 361         }
 362 }
 363
 364 #if defined(RGX_FEATURE_META_DMA)
 365 IMG_VOID RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR          *psDest,
 366                                                           DEVMEM_MEMDESC                *psSrcMemDesc,
 367                                                           RGXFWIF_DEV_VIRTADDR  *psSrcFWDevVAddr,
 368                                                           IMG_UINT32                    uiOffset)
 369 {
 370         PVRSRV_ERROR            eError;
 371         IMG_DEV_VIRTADDR        sDevVirtAddr;
 372
 373         eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
 374         PVR_ASSERT(eError == PVRSRV_OK);
 375
 376         psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
 377         psDest->psDevVirtAddr.uiAddr += uiOffset;
 378         psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
 379
 380         DevmemReleaseDevVirtAddr(psSrcMemDesc);
 381 }
 382 #endif
 383
 384 IMG_VOID RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
 385 {
 386         DevmemReleaseDevVirtAddr(psSrc);
 387 }
 388
 389 struct _RGX_SERVER_COMMON_CONTEXT_ {
 390         DEVMEM_MEMDESC *psFWCommonContextMemDesc;
 391         PRGXFWIF_FWCOMMONCONTEXT sFWCommonContextFWAddr;
 392         DEVMEM_MEMDESC *psFWMemContextMemDesc;
 393         DEVMEM_MEMDESC *psFWFrameworkMemDesc;
 394         DEVMEM_MEMDESC *psContextStateMemDesc;
 395         RGX_CLIENT_CCB *psClientCCB;
 396         DEVMEM_MEMDESC *psClientCCBMemDesc;
 397         DEVMEM_MEMDESC *psClientCCBCtrlMemDesc;
 398         IMG_BOOL bCommonContextMemProvided;
 399         IMG_UINT32 ui32ContextID;
 400         DLLIST_NODE sListNode;
 401         RGXFWIF_CONTEXT_RESET_REASON eLastResetReason;
 402 };
 403
 404 PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
 405                                                                          PVRSRV_DEVICE_NODE *psDeviceNode,
 406                                                                          const IMG_CHAR *pszContextName,
 407                                                                          DEVMEM_MEMDESC *psAllocatedMemDesc,
 408                                                                          IMG_UINT32 ui32AllocatedOffset,
 409                                                                          DEVMEM_MEMDESC *psFWMemContextMemDesc,
 410                                                                          DEVMEM_MEMDESC *psContextStateMemDesc,
 411                                                                          IMG_UINT32 ui32CCBAllocSize,
 412                                                                          IMG_UINT32 ui32Priority,
 413                                                                          RGX_COMMON_CONTEXT_INFO *psInfo,
 414                                                                          RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext)
 415 {
 416         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
 417         RGX_SERVER_COMMON_CONTEXT *psServerCommonContext;
 418         RGXFWIF_FWCOMMONCONTEXT *psFWCommonContext;
 419         IMG_UINT32 ui32FWCommonContextOffset;
 420         IMG_UINT8 *pui8Ptr;
 421         PVRSRV_ERROR eError;
 422
 423         /*
 424                 Allocate all the resources that are required
 425         */
 426         psServerCommonContext = OSAllocMem(sizeof(*psServerCommonContext));
 427         if (psServerCommonContext == IMG_NULL)
 428         {
 429                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 430                 goto fail_alloc;
 431         }
 432
 433         if (psAllocatedMemDesc)
 434         {
 435                 PDUMPCOMMENT("Using existing MemDesc for Rogue firmware %s context (offset = %d)",
 436                                          pszContextName,
 437                                          ui32AllocatedOffset);
 438                 ui32FWCommonContextOffset = ui32AllocatedOffset;
 439                 psServerCommonContext->psFWCommonContextMemDesc = psAllocatedMemDesc;
 440                 psServerCommonContext->bCommonContextMemProvided = IMG_TRUE;
 441         }
 442         else
 443         {
 444                 /* Allocate device memory for the firmware context */
 445                 PDUMPCOMMENT("Allocate Rogue firmware %s context", pszContextName);
 446                 eError = DevmemFwAllocate(psDevInfo,
 447                                                                 sizeof(*psFWCommonContext),
 448                                                                 RGX_FWCOMCTX_ALLOCFLAGS,
 449                                                                 "FirmwareContext",
 450                                                                 &psServerCommonContext->psFWCommonContextMemDesc);
 451                 if (eError != PVRSRV_OK)
 452                 {
 453                         PVR_DPF((PVR_DBG_ERROR,"%s : Failed to allocate firmware %s context (%s)",
 454                                                                         __FUNCTION__,
 455                                                                         pszContextName,
 456                                                                         PVRSRVGetErrorStringKM(eError)));
 457                         goto fail_contextalloc;
 458                 }
 459                 ui32FWCommonContextOffset = 0;
 460                 psServerCommonContext->bCommonContextMemProvided = IMG_FALSE;
 461         }
 462
 463         /* Record this context so we can refer to it if the FW needs to tell us it was reset. */
 464         psServerCommonContext->eLastResetReason = RGXFWIF_CONTEXT_RESET_REASON_NONE;
 465         psServerCommonContext->ui32ContextID    = psDevInfo->ui32CommonCtxtCurrentID++;
 466         dllist_add_to_tail(&(psDevInfo->sCommonCtxtListHead), &(psServerCommonContext->sListNode));
 467
 468         /* Allocate the client CCB */
 469         eError = RGXCreateCCB(psDeviceNode,
 470                                                   ui32CCBAllocSize,
 471                                                   psConnection,
 472                                                   pszContextName,
 473                                                   psServerCommonContext,
 474                                                   &psServerCommonContext->psClientCCB,
 475                                                   &psServerCommonContext->psClientCCBMemDesc,
 476                                                   &psServerCommonContext->psClientCCBCtrlMemDesc);
 477         if (eError != PVRSRV_OK)
 478         {
 479                 PVR_DPF((PVR_DBG_ERROR, "%s: failed to create CCB for %s context(%s)",
 480                                                                 __FUNCTION__,
 481                                                                 pszContextName,
 482                                                                 PVRSRVGetErrorStringKM(eError)));
 483                 goto fail_allocateccb;
 484         }
 485
 486         /*
 487                 Temporarily map the firmware context to the kernel and init it
 488         */
 489         eError = DevmemAcquireCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc,
 490                                       (IMG_VOID **)&pui8Ptr);
 491         if (eError != PVRSRV_OK)
 492         {
 493                 PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware %s context (%s)to CPU",
 494                                                                 __FUNCTION__,
 495                                                                 pszContextName,
 496                                                                 PVRSRVGetErrorStringKM(eError)));
 497                 goto fail_cpuvirtacquire;
 498         }
 499
 500         psFWCommonContext = (RGXFWIF_FWCOMMONCONTEXT *) (pui8Ptr + ui32FWCommonContextOffset);
 501
 502         /* Set the firmware CCB device addresses in the firmware common context */
 503         RGXSetFirmwareAddress(&psFWCommonContext->psCCB,
 504                                                   psServerCommonContext->psClientCCBMemDesc,
 505                                                   0, RFW_FWADDR_FLAG_NONE);
 506         RGXSetFirmwareAddress(&psFWCommonContext->psCCBCtl,
 507                                                   psServerCommonContext->psClientCCBCtrlMemDesc,
 508                                                   0, RFW_FWADDR_FLAG_NONE);
 509
 510         /* Set the memory context device address */
 511         psServerCommonContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
 512         RGXSetFirmwareAddress(&psFWCommonContext->psFWMemContext,
 513                                                   psFWMemContextMemDesc,
 514                                                   0, RFW_FWADDR_FLAG_NONE);
 515
 516         /* Set the framework register updates address */
 517         psServerCommonContext->psFWFrameworkMemDesc = psInfo->psFWFrameworkMemDesc;
 518         RGXSetFirmwareAddress(&psFWCommonContext->psRFCmd,
 519                                                   psInfo->psFWFrameworkMemDesc,
 520                                                   0, RFW_FWADDR_FLAG_NONE);
 521
 522         psFWCommonContext->ui32Priority = ui32Priority;
 523         psFWCommonContext->ui32PrioritySeqNum = 0;
 524
 525         if(psInfo->psMCUFenceAddr != IMG_NULL)
 526         {
 527                 psFWCommonContext->ui64MCUFenceAddr = psInfo->psMCUFenceAddr->uiAddr;
 528         }
 529
 530         /* Store a references to Server Common Context and PID for notifications back from the FW. */
 531         psFWCommonContext->ui32ServerCommonContextID = psServerCommonContext->ui32ContextID;
 532         psFWCommonContext->ui32PID                   = OSGetCurrentProcessID();
 533
 534         /* Set the firmware GPU context state buffer */
 535         psServerCommonContext->psContextStateMemDesc = psContextStateMemDesc;
 536         if (psContextStateMemDesc)
 537         {
 538                 RGXSetFirmwareAddress(&psFWCommonContext->psContextState,
 539                                                           psContextStateMemDesc,
 540                                                           0,
 541                                                           RFW_FWADDR_FLAG_NONE);
 542         }
 543
 544         /*
 545          * Dump the created context
 546          */
 547         PDUMPCOMMENT("Dump %s context", pszContextName);
 548         DevmemPDumpLoadMem(psServerCommonContext->psFWCommonContextMemDesc,
 549                                            ui32FWCommonContextOffset,
 550                                            sizeof(*psFWCommonContext),
 551                                            PDUMP_FLAGS_CONTINUOUS);
 552
 553         /* We've finished the setup so release the CPU mapping */
 554         DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
 555
 556         /* Map this allocation into the FW */
 557         RGXSetFirmwareAddress(&psServerCommonContext->sFWCommonContextFWAddr,
 558                                                   psServerCommonContext->psFWCommonContextMemDesc,
 559                                                   ui32FWCommonContextOffset,
 560                                                   RFW_FWADDR_FLAG_NONE);
 561
 562 #if defined(LINUX)
 563         trace_rogue_create_fw_context(OSGetCurrentProcessName(),
 564                                                                   pszContextName,
 565                                                                   psServerCommonContext->sFWCommonContextFWAddr.ui32Addr);
 566 #endif
 567
 568         *ppsServerCommonContext = psServerCommonContext;
 569         return PVRSRV_OK;
 570
 571 fail_allocateccb:
 572         DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
 573 fail_cpuvirtacquire:
 574         RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
 575         if (!psServerCommonContext->bCommonContextMemProvided)
 576         {
 577                 DevmemFwFree(psServerCommonContext->psFWCommonContextMemDesc);
 578         }
 579 fail_contextalloc:
 580         OSFreeMem(psServerCommonContext);
 581 fail_alloc:
 582         return eError;
 583 }
 584
 585 IMG_VOID FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 586 {
 587         /*
 588                 Unmap the context itself and then all it's resources
 589         */
 590
 591         /* Unmap the FW common context */
 592         RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
 593         /* Umap context state buffer (if there was one) */
 594         if (psServerCommonContext->psContextStateMemDesc)
 595         {
 596                 RGXUnsetFirmwareAddress(psServerCommonContext->psContextStateMemDesc);
 597         }
 598         /* Unmap the framework buffer */
 599         RGXUnsetFirmwareAddress(psServerCommonContext->psFWFrameworkMemDesc);
 600         /* Unmap client CCB and CCB control */
 601         RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
 602         RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
 603         /* Unmap the memory context */
 604         RGXUnsetFirmwareAddress(psServerCommonContext->psFWMemContextMemDesc);
 605
 606         /* Destroy the client CCB */
 607         RGXDestroyCCB(psServerCommonContext->psClientCCB);
 608
 609         /* Remove the context from the list of all contexts. */
 610         dllist_remove_node(&psServerCommonContext->sListNode);
 611
 612         /* Free the FW common context (if there was one) */
 613         if (!psServerCommonContext->bCommonContextMemProvided)
 614         {
 615                 DevmemFwFree(psServerCommonContext->psFWCommonContextMemDesc);
 616         }
 617         /* Free the hosts representation of the common context */
 618         OSFreeMem(psServerCommonContext);
 619 }
 620
 621 PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 622 {
 623         return psServerCommonContext->sFWCommonContextFWAddr;
 624 }
 625
 626 RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 627 {
 628         return psServerCommonContext->psClientCCB;
 629 }
 630
 631 RGXFWIF_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
 632 {
 633         RGXFWIF_CONTEXT_RESET_REASON  eLastResetReason;
 634
 635         PVR_ASSERT(psServerCommonContext != IMG_NULL);
 636
 637         /* Take the most recent reason and reset for next time... */
 638         eLastResetReason = psServerCommonContext->eLastResetReason;
 639         psServerCommonContext->eLastResetReason = RGXFWIF_CONTEXT_RESET_REASON_NONE;
 640
 641         return eLastResetReason;
 642 }
 643
 644 /*!
 645 *******************************************************************************
 646  @Function              RGXFreeKernelCCB
 647  @Description   Free a kernel CCB
 648  @Input                 psDevInfo
 649  @Input                 eKCCBType
 650
 651  @Return                PVRSRV_ERROR
 652 ******************************************************************************/
 653 static IMG_VOID RGXFreeKernelCCB(PVRSRV_RGXDEV_INFO     *psDevInfo,
 654                                                                  RGXFWIF_DM                             eKCCBType)
 655 {
 656         if (psDevInfo->apsKernelCCBMemDesc[eKCCBType] != IMG_NULL)
 657         {
 658                 if (psDevInfo->apsKernelCCB[eKCCBType] != IMG_NULL)
 659                 {
 660                         DevmemReleaseCpuVirtAddr(psDevInfo->apsKernelCCBMemDesc[eKCCBType]);
 661                         psDevInfo->apsKernelCCB[eKCCBType] = IMG_NULL;
 662                 }
 663                 DevmemFwFree(psDevInfo->apsKernelCCBMemDesc[eKCCBType]);
 664                 psDevInfo->apsKernelCCBMemDesc[eKCCBType] = IMG_NULL;
 665         }
 666         if (psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType] != IMG_NULL)
 667         {
 668                 if (psDevInfo->apsKernelCCBCtl[eKCCBType] != IMG_NULL)
 669                 {
 670                         DevmemReleaseCpuVirtAddr(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType]);
 671                         psDevInfo->apsKernelCCBCtl[eKCCBType] = IMG_NULL;
 672                 }
 673                 DevmemFwFree(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType]);
 674                 psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType] = IMG_NULL;
 675         }
 676 }
 677
 678 /*!
 679 *******************************************************************************
 680  @Function              RGXSetupKernelCCB
 681  @Description   Allocate and initialise a kernel CCB
 682  @Input                 psDevInfo
 683
 684  @Return                PVRSRV_ERROR
 685 ******************************************************************************/
 686 static PVRSRV_ERROR RGXSetupKernelCCB(PVRSRV_RGXDEV_INFO        *psDevInfo,
 687                                                                           RGXFWIF_INIT                  *psRGXFWInit,
 688                                                                           RGXFWIF_DM                    eKCCBType,
 689                                                                           IMG_UINT32                    ui32NumCmdsLog2,
 690                                                                           IMG_UINT32                    ui32CmdSize)
 691 {
 692         PVRSRV_ERROR            eError;
 693         RGXFWIF_CCB_CTL         *psKCCBCtl;
 694         DEVMEM_FLAGS_T          uiCCBCtlMemAllocFlags, uiCCBMemAllocFlags;
 695         IMG_UINT32                      ui32kCCBSize = (1U << ui32NumCmdsLog2);
 696
 697
 698         /*
 699          * FIXME: the write offset need not be writeable by the firmware, indeed may
 700          * not even be needed for reading. Consider moving it to its own data
 701          * structure.
 702          */
 703         uiCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 704                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 705                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 706                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 707                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 708                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 709                                                         PVRSRV_MEMALLOCFLAG_UNCACHED |
 710                                                          PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 711
 712         /* Allocation flags for Kernel CCB */
 713         uiCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 714                                                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) |
 715                                                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 716                                                  PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 717                                                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 718                                                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 719                                                  PVRSRV_MEMALLOCFLAG_UNCACHED |
 720                                                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 721
 722         /*
 723                 Allocate memory for the kernel CCB control.
 724         */
 725         PDUMPCOMMENT("Allocate memory for kernel CCB control %u", eKCCBType);
 726         eError = DevmemFwAllocate(psDevInfo,
 727                                                         sizeof(RGXFWIF_CCB_CTL),
 728                                                         uiCCBCtlMemAllocFlags,
 729                                                         "KernelCCBControl",
 730                             &psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType]);
 731
 732         if (eError != PVRSRV_OK)
 733         {
 734                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to allocate kernel CCB ctl %u (%u)",
 735                                 eKCCBType, eError));
 736                 goto fail;
 737         }
 738
 739         /*
 740                 Allocate memory for the kernel CCB.
 741                 (this will reference further command data in non-shared CCBs)
 742         */
 743         PDUMPCOMMENT("Allocate memory for kernel CCB %u", eKCCBType);
 744         eError = DevmemFwAllocate(psDevInfo,
 745                                                         ui32kCCBSize * ui32CmdSize,
 746                                                         uiCCBMemAllocFlags,
 747                                                         "KernelCCB",
 748                             &psDevInfo->apsKernelCCBMemDesc[eKCCBType]);
 749
 750         if (eError != PVRSRV_OK)
 751         {
 752                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to allocate kernel CCB %u (%u)",
 753                                 eKCCBType, eError));
 754                 goto fail;
 755         }
 756
 757         /*
 758                 Map the kernel CCB control to the kernel.
 759         */
 760         eError = DevmemAcquireCpuVirtAddr(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
 761                                       (IMG_VOID **)&psDevInfo->apsKernelCCBCtl[eKCCBType]);
 762         if (eError != PVRSRV_OK)
 763         {
 764                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to acquire cpu kernel CCB Ctl %u (%u)",
 765                                 eKCCBType, eError));
 766                 goto fail;
 767         }
 768
 769         /*
 770                 Map the kernel CCB to the kernel.
 771         */
 772         eError = DevmemAcquireCpuVirtAddr(psDevInfo->apsKernelCCBMemDesc[eKCCBType],
 773                                       (IMG_VOID **)&psDevInfo->apsKernelCCB[eKCCBType]);
 774         if (eError != PVRSRV_OK)
 775         {
 776                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupKernelCCB: Failed to acquire cpu kernel CCB %u (%u)",
 777                                 eKCCBType, eError));
 778                 goto fail;
 779         }
 780
 781         /*
 782          * Initialise the kernel CCB control.
 783          */
 784         psKCCBCtl = psDevInfo->apsKernelCCBCtl[eKCCBType];
 785         psKCCBCtl->ui32WriteOffset = 0;
 786         psKCCBCtl->ui32ReadOffset = 0;
 787         psKCCBCtl->ui32WrapMask = ui32kCCBSize - 1;
 788         psKCCBCtl->ui32CmdSize = ui32CmdSize;
 789
 790         /*
 791          * Set-up RGXFWIfCtl pointers to access the kCCBs
 792          */
 793         RGXSetFirmwareAddress(&psRGXFWInit->psKernelCCBCtl[eKCCBType],
 794                                                   psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
 795                                                   0, RFW_FWADDR_NOREF_FLAG);
 796
 797         RGXSetFirmwareAddress(&psRGXFWInit->psKernelCCB[eKCCBType],
 798                                                   psDevInfo->apsKernelCCBMemDesc[eKCCBType],
 799                                                   0, RFW_FWADDR_NOREF_FLAG);
 800
 801         psRGXFWInit->eDM[eKCCBType] = eKCCBType;
 802
 803         /*
 804          * Pdump the kernel CCB control.
 805          */
 806         PDUMPCOMMENT("Initialise kernel CCB ctl %d", eKCCBType);
 807         DevmemPDumpLoadMem(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
 808                                            0,
 809                                            sizeof(RGXFWIF_CCB_CTL),
 810                                            0);
 811
 812         return PVRSRV_OK;
 813
 814 fail:
 815         RGXFreeKernelCCB(psDevInfo, eKCCBType);
 816
 817         PVR_ASSERT(eError != PVRSRV_OK);
 818         return eError;
 819 }
 820
 821 /*!
 822 *******************************************************************************
 823  @Function              RGXFreeFirmwareCCB
 824  @Description   Free a firmware CCB
 825  @Input                 psDevInfo
 826  @Input                 eFWCCBType
 827
 828  @Return                PVRSRV_ERROR
 829 ******************************************************************************/
 830 static IMG_VOID RGXFreeFirmwareCCB(PVRSRV_RGXDEV_INFO   *psDevInfo,
 831                                                                  RGXFWIF_DM                             eFWCCBType)
 832 {
 833         if (psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType] != IMG_NULL)
 834         {
 835                 if (psDevInfo->apsFirmwareCCB[eFWCCBType] != IMG_NULL)
 836                 {
 837                         DevmemReleaseCpuVirtAddr(psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType]);
 838                         psDevInfo->apsFirmwareCCB[eFWCCBType] = IMG_NULL;
 839                 }
 840                 DevmemFwFree(psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType]);
 841                 psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType] = IMG_NULL;
 842         }
 843         if (psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType] != IMG_NULL)
 844         {
 845                 if (psDevInfo->apsFirmwareCCBCtl[eFWCCBType] != IMG_NULL)
 846                 {
 847                         DevmemReleaseCpuVirtAddr(psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType]);
 848                         psDevInfo->apsFirmwareCCBCtl[eFWCCBType] = IMG_NULL;
 849                 }
 850                 DevmemFwFree(psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType]);
 851                 psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType] = IMG_NULL;
 852         }
 853 }
 854
 855 /*!
 856 *******************************************************************************
 857  @Function              RGXSetupFirmwareCCB
 858  @Description   Allocate and initialise a Firmware CCB
 859  @Input                 psDevInfo
 860
 861  @Return                PVRSRV_ERROR
 862 ******************************************************************************/
 863 static PVRSRV_ERROR RGXSetupFirmwareCCB(PVRSRV_RGXDEV_INFO      *psDevInfo,
 864                                                                           RGXFWIF_INIT                  *psRGXFWInit,
 865                                                                           RGXFWIF_DM                    eFWCCBType,
 866                                                                           IMG_UINT32                    ui32NumCmdsLog2,
 867                                                                           IMG_UINT32                    ui32CmdSize)
 868 {
 869         PVRSRV_ERROR            eError;
 870         RGXFWIF_CCB_CTL         *psFWCCBCtl;
 871         DEVMEM_FLAGS_T          uiCCBCtlMemAllocFlags, uiCCBMemAllocFlags;
 872         IMG_UINT32                      ui32FWCCBSize = (1U << ui32NumCmdsLog2);
 873
 874         /*
 875          * FIXME: the write offset need not be writeable by the host, indeed may
 876          * not even be needed for reading. Consider moving it to its own data
 877          * structure.
 878          */
 879         uiCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 880                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 881                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 882                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 883                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
 884                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 885                                                         PVRSRV_MEMALLOCFLAG_UNCACHED |
 886                                                          PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 887
 888         /* Allocation flags for Firmware CCB */
 889         uiCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 890                                                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
 891                                                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
 892                                                  PVRSRV_MEMALLOCFLAG_CPU_READABLE |
 893                                                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
 894                                                  PVRSRV_MEMALLOCFLAG_UNCACHED |
 895                                                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
 896
 897         /*
 898                 Allocate memory for the Firmware CCB control.
 899         */
 900         PDUMPCOMMENT("Allocate memory for firmware CCB control %u", eFWCCBType);
 901         eError = DevmemFwAllocate(psDevInfo,
 902                                                         sizeof(RGXFWIF_CCB_CTL),
 903                                                         uiCCBCtlMemAllocFlags,
 904                                                         "FirmwareCCBControl",
 905                             &psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType]);
 906
 907         if (eError != PVRSRV_OK)
 908         {
 909                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to allocate Firmware CCB ctl %u (%u)",
 910                                 eFWCCBType, eError));
 911                 goto fail;
 912         }
 913
 914         /*
 915                 Allocate memory for the Firmware CCB.
 916                 (this will reference further command data in non-shared CCBs)
 917         */
 918         PDUMPCOMMENT("Allocate memory for firmware CCB %u", eFWCCBType);
 919         eError = DevmemFwAllocate(psDevInfo,
 920                                                         ui32FWCCBSize * ui32CmdSize,
 921                                                         uiCCBMemAllocFlags,
 922                                                         "FirmwareCCB",
 923                             &psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType]);
 924
 925         if (eError != PVRSRV_OK)
 926         {
 927                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to allocate Firmware CCB %u (%u)",
 928                                 eFWCCBType, eError));
 929                 goto fail;
 930         }
 931
 932         /*
 933                 Map the Firmware CCB control to the kernel.
 934         */
 935         eError = DevmemAcquireCpuVirtAddr(psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType],
 936                                       (IMG_VOID **)&psDevInfo->apsFirmwareCCBCtl[eFWCCBType]);
 937         if (eError != PVRSRV_OK)
 938         {
 939                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to acquire cpu firmware CCB Ctl %u (%u)",
 940                                 eFWCCBType, eError));
 941                 goto fail;
 942         }
 943
 944         /*
 945                 Map the firmware CCB to the kernel.
 946         */
 947         eError = DevmemAcquireCpuVirtAddr(psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType],
 948                                       (IMG_VOID **)&psDevInfo->apsFirmwareCCB[eFWCCBType]);
 949         if (eError != PVRSRV_OK)
 950         {
 951                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmwareCCB: Failed to acquire cpu firmware CCB %u (%u)",
 952                                 eFWCCBType, eError));
 953                 goto fail;
 954         }
 955
 956         /*
 957          * Initialise the firmware CCB control.
 958          */
 959         psFWCCBCtl = psDevInfo->apsFirmwareCCBCtl[eFWCCBType];
 960         psFWCCBCtl->ui32WriteOffset = 0;
 961         psFWCCBCtl->ui32ReadOffset = 0;
 962         psFWCCBCtl->ui32WrapMask = ui32FWCCBSize - 1;
 963         psFWCCBCtl->ui32CmdSize = ui32CmdSize;
 964
 965         /*
 966          * Set-up RGXFWIfCtl pointers to access the kCCBs
 967          */
 968         RGXSetFirmwareAddress(&psRGXFWInit->psFirmwareCCBCtl[eFWCCBType],
 969                                                   psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType],
 970                                                   0, RFW_FWADDR_NOREF_FLAG);
 971
 972         RGXSetFirmwareAddress(&psRGXFWInit->psFirmwareCCB[eFWCCBType],
 973                                                   psDevInfo->apsFirmwareCCBMemDesc[eFWCCBType],
 974                                                   0, RFW_FWADDR_NOREF_FLAG);
 975
 976         psRGXFWInit->eDM[eFWCCBType] = eFWCCBType;
 977
 978         /*
 979          * Pdump the kernel CCB control.
 980          */
 981         PDUMPCOMMENT("Initialise firmware CCB ctl %d", eFWCCBType);
 982         DevmemPDumpLoadMem(psDevInfo->apsFirmwareCCBCtlMemDesc[eFWCCBType],
 983                                            0,
 984                                            sizeof(RGXFWIF_CCB_CTL),
 985                                            0);
 986
 987         return PVRSRV_OK;
 988
 989 fail:
 990         RGXFreeFirmwareCCB(psDevInfo, eFWCCBType);
 991
 992         PVR_ASSERT(eError != PVRSRV_OK);
 993         return eError;
 994 }
 995
 996 static IMG_VOID RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
 997 {
 998         PMR *psPMR;
 999
1000         if (psDevInfo->psRGXFaultAddressMemDesc)
1001         {
1002                 if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc,(IMG_VOID **)&psPMR) == PVRSRV_OK)
1003                 {
1004                         PMRUnlockSysPhysAddresses(psPMR);
1005                 }
1006                 DevmemFwFree(psDevInfo->psRGXFaultAddressMemDesc);
1007                 psDevInfo->psRGXFaultAddressMemDesc = IMG_NULL;
1008         }
1009 }
1010
1011 static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE        *psDeviceNode, RGXFWIF_INIT *psRGXFWInit)
1012 {
1013         PVRSRV_ERROR            eError = PVRSRV_OK;
1014         IMG_UINT32                      *pui32MemoryVirtAddr;
1015         IMG_UINT32                      i;
1016         IMG_SIZE_T                      ui32PageSize;
1017         DEVMEM_FLAGS_T          uiMemAllocFlags;
1018         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1019         PMR                                     *psPMR;
1020
1021         ui32PageSize = OSGetPageSize();
1022
1023         /* Allocate page of memory to use for page faults on non-blocking memory transactions */
1024         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1025                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1026                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1027                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1028                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1029                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1030                                                 PVRSRV_MEMALLOCFLAG_UNCACHED;
1031
1032         psDevInfo->psRGXFaultAddressMemDesc = IMG_NULL;
1033         eError = DevmemFwAllocateExportable(psDeviceNode,
1034                                                                                 ui32PageSize,
1035                                                                                 uiMemAllocFlags,
1036                                                                                 "FaultAddress",
1037                                                                                 &psDevInfo->psRGXFaultAddressMemDesc);
1038
1039         if (eError != PVRSRV_OK)
1040         {
1041                 PVR_DPF((PVR_DBG_ERROR,"Failed to allocate mem for fault address (%u)",
1042                                 eError));
1043                 goto failFaultAddressDescAlloc;
1044         }
1045
1046         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
1047                                                                           (IMG_VOID **)&pui32MemoryVirtAddr);
1048         if (eError != PVRSRV_OK)
1049         {
1050                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire mem for fault adress (%u)",
1051                                 eError));
1052                 goto failFaultAddressDescAqCpuVirt;
1053         }
1054
1055         for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
1056         {
1057                 *(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
1058         }
1059
1060         eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc,(IMG_VOID **)&psPMR);
1061
1062         if (eError != PVRSRV_OK)
1063         {
1064                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error getting PMR for fault adress (%u)",
1065                                 eError));
1066
1067                 goto failFaultAddressDescGetPMR;
1068         }
1069         else
1070         {
1071                 IMG_BOOL bValid;
1072                 IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
1073
1074                 eError = PMRLockSysPhysAddresses(psPMR,ui32Log2PageSize);
1075
1076                 if (eError != PVRSRV_OK)
1077                 {
1078                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error locking physical address for fault address MemDesc (%u)",
1079                                         eError));
1080
1081                         goto failFaultAddressDescLockPhys;
1082                 }
1083
1084                 eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize,1,0,&(psRGXFWInit->sFaultPhysAddr),&bValid);
1085
1086                 if (eError != PVRSRV_OK)
1087                 {
1088                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Error getting physical address for fault address MemDesc (%u)",
1089                                         eError));
1090
1091                         goto failFaultAddressDescGetPhys;
1092                 }
1093
1094                 if (!bValid)
1095                 {
1096                         psRGXFWInit->sFaultPhysAddr.uiAddr = 0;
1097                         PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed getting physical address for fault address MemDesc - invalid page (0x%llX)",
1098                                         psRGXFWInit->sFaultPhysAddr.uiAddr));
1099
1100                         goto failFaultAddressDescGetPhys;
1101                 }
1102         }
1103
1104         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
1105
1106         return PVRSRV_OK;
1107
1108 failFaultAddressDescGetPhys:
1109         PMRUnlockSysPhysAddresses(psPMR);
1110
1111 failFaultAddressDescLockPhys:
1112
1113 failFaultAddressDescGetPMR:
1114         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
1115
1116 failFaultAddressDescAqCpuVirt:
1117         DevmemFwFree(psDevInfo->psRGXFaultAddressMemDesc);
1118         psDevInfo->psRGXFaultAddressMemDesc = IMG_NULL;
1119
1120 failFaultAddressDescAlloc:
1121
1122         return eError;
1123 }
1124
1125 static PVRSRV_ERROR RGXHwBrn37200(PVRSRV_RGXDEV_INFO *psDevInfo)
1126 {
1127         PVRSRV_ERROR                    eError = PVRSRV_OK;
1128
1129 #if defined(FIX_HW_BRN_37200)
1130         struct _DEVMEM_HEAP_    *psBRNHeap;
1131         DEVMEM_FLAGS_T                  uiFlags;
1132         IMG_DEV_VIRTADDR                sTmpDevVAddr;
1133         IMG_SIZE_T                              uiPageSize;
1134
1135         uiPageSize = OSGetPageSize();
1136
1137         uiFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1138                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1139                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1140                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1141                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1142                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1143                                 PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1144                                 PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1145                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1146
1147         eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx,
1148                                                           "HWBRN37200", /* FIXME: We need to create an IDENT macro for this string.
1149                                                                          Make sure the IDENT macro is not accessible to userland */
1150                                                           &psBRNHeap);
1151
1152         if (eError != PVRSRV_OK)
1153         {
1154                 PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: HWBRN37200 Failed DevmemFindHeapByName (%u)", eError));
1155                 goto failFWHWBRN37200FindHeapByName;
1156         }
1157
1158         psDevInfo->psRGXFWHWBRN37200MemDesc = IMG_NULL;
1159         eError = DevmemAllocate(psBRNHeap,
1160                                                 uiPageSize,
1161                                                 ROGUE_CACHE_LINE_SIZE,
1162                                                 uiFlags,
1163                                                 "HWBRN37200",
1164                                                 &psDevInfo->psRGXFWHWBRN37200MemDesc);
1165
1166         if (eError != PVRSRV_OK)
1167         {
1168                 PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: Failed to allocate %u bytes for HWBRN37200 (%u)",
1169                                 (IMG_UINT32)uiPageSize,
1170                                 eError));
1171                 goto failFWHWBRN37200MemDescAlloc;
1172         }
1173
1174         /*
1175                 We need to map it so the heap for this allocation
1176                 is set
1177         */
1178         eError = DevmemMapToDevice(psDevInfo->psRGXFWHWBRN37200MemDesc,
1179                                                    psBRNHeap,
1180                                                    &sTmpDevVAddr);
1181
1182         if (eError != PVRSRV_OK)
1183         {
1184                 PVR_DPF((PVR_DBG_ERROR,"RGXHwBrn37200: Failed to allocate %u bytes for HWBRN37200 (%u)",
1185                                 (IMG_UINT32)uiPageSize,
1186                                 eError));
1187                 goto failFWHWBRN37200DevmemMapToDevice;
1188         }
1189
1190         return PVRSRV_OK;
1191
1192 failFWHWBRN37200DevmemMapToDevice:
1193
1194 failFWHWBRN37200MemDescAlloc:
1195         DevmemFwFree(psDevInfo->psRGXFWHWBRN37200MemDesc);
1196         psDevInfo->psRGXFWHWBRN37200MemDesc = IMG_NULL;
1197
1198 failFWHWBRN37200FindHeapByName:
1199 #endif
1200
1201         return eError;
1202 }
1203
1204 /*!
1205 *******************************************************************************
1206
1207  @Function      RGXSetupFirmware
1208
1209  @Description
1210
1211  Setups all the firmware related data
1212
1213  @Input psDevInfo
1214
1215  @Return PVRSRV_ERROR
1216
1217 ******************************************************************************/
1218 PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE        *psDeviceNode,
1219                                                              IMG_BOOL                   bEnableSignatureChecks,
1220                                                              IMG_UINT32                 ui32SignatureChecksBufSize,
1221                                                              IMG_UINT32                 ui32HWPerfFWBufSizeKB,
1222                                                              IMG_UINT64                 ui64HWPerfFilter,
1223                                                              IMG_UINT32                 ui32RGXFWAlignChecksSize,
1224                                                              IMG_UINT32                 *pui32RGXFWAlignChecks,
1225                                                              IMG_UINT32                 ui32ConfigFlags,
1226                                                              IMG_UINT32                 ui32LogType,
1227                                                              IMG_UINT32            ui32NumTilingCfgs,
1228                                                              IMG_UINT32            *pui32BIFTilingXStrides,
1229                                                              IMG_UINT32                 ui32FilterFlags,
1230                                                              IMG_UINT32                 ui32JonesDisableMask,
1231                                                              IMG_UINT32                 ui32HWRDebugDumpLimit,
1232                                                                  IMG_UINT32                     ui32HWPerfCountersDataSize,
1233                                                              RGXFWIF_DEV_VIRTADDR       *psRGXFWInitFWAddr,
1234                                                              RGX_RD_POWER_ISLAND_CONF eRGXRDPowerIslandConf)
1235
1236 {
1237         PVRSRV_ERROR            eError;
1238         DEVMEM_FLAGS_T          uiMemAllocFlags;
1239         RGXFWIF_INIT            *psRGXFWInit;
1240         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1241         IMG_UINT32                      dm;
1242 #if defined(RGX_FEATURE_META_DMA)
1243         RGXFWIF_DEV_VIRTADDR sRGXTmpCorememDataStoreFWAddr;
1244 #endif
1245
1246         /* Fw init data */
1247         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1248                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) |
1249                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1250                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1251                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1252                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1253                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1254                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1255                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1256                                                 /* FIXME: Change to Cached */
1257
1258         PDUMPCOMMENT("Allocate RGXFWIF_INIT structure");
1259         eError = DevmemFwAllocate(psDevInfo,
1260                                                         sizeof(RGXFWIF_INIT),
1261                                                         uiMemAllocFlags,
1262                                                         "FirmwareInitStructure",
1263                                                         &psDevInfo->psRGXFWIfInitMemDesc);
1264
1265         if (eError != PVRSRV_OK)
1266         {
1267                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw if ctl (%u)",
1268                                 (IMG_UINT32)sizeof(RGXFWIF_INIT),
1269                                 eError));
1270                 goto fail;
1271         }
1272
1273         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc,
1274                                                                           (IMG_VOID **)&psRGXFWInit);
1275         if (eError != PVRSRV_OK)
1276         {
1277                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel fw if ctl (%u)",
1278                                 eError));
1279                 goto fail;
1280         }
1281
1282         RGXSetFirmwareAddress(psRGXFWInitFWAddr,
1283                                                 psDevInfo->psRGXFWIfInitMemDesc,
1284                                                 0, RFW_FWADDR_NOREF_FLAG);
1285
1286         /* FW Trace buffer */
1287         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1288                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1289                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1290                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1291                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1292                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1293                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1294
1295         PDUMPCOMMENT("Allocate rgxfw trace structure");
1296         eError = DevmemFwAllocate(psDevInfo,
1297                                                         sizeof(RGXFWIF_TRACEBUF),
1298                                                         uiMemAllocFlags,
1299                                                         "FirmwareTraceStructure",
1300                                                         &psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
1301
1302         if (eError != PVRSRV_OK)
1303         {
1304                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw trace (%u)",
1305                                 (IMG_UINT32)sizeof(RGXFWIF_TRACEBUF),
1306                                 eError));
1307                 goto fail;
1308         }
1309
1310         RGXSetFirmwareAddress(&psRGXFWInit->psTraceBufCtl,
1311                                                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
1312                                                 0, RFW_FWADDR_NOREF_FLAG);
1313
1314         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
1315                                                                           (IMG_VOID **)&psDevInfo->psRGXFWIfTraceBuf);
1316         if (eError != PVRSRV_OK)
1317         {
1318                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel tracebuf ctl (%u)",
1319                                 eError));
1320                 goto fail;
1321         }
1322
1323         /* Determine the size of the HWPerf FW buffer */
1324         if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX>>10))
1325         {
1326                 /* Size specified as a AppHint but it is too big */
1327                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
1328                                 ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX>>10));
1329                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_MAX;
1330
1331         }
1332         else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN>>10))
1333         {
1334                 /* Size specified as in AppHint HWPerfFWBufSizeInKB */
1335                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: Using HWPerf FW buffer size of %u KB",
1336                                 ui32HWPerfFWBufSizeKB));
1337                 psDevInfo->ui32RGXFWIfHWPerfBufSize = ui32HWPerfFWBufSizeKB<<10;
1338         }
1339         else if (ui32HWPerfFWBufSizeKB > 0)
1340         {
1341                 /* Size specified as a AppHint but it is too small */
1342                 PVR_DPF((PVR_DBG_WARNING,"RGXSetupFirmware: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
1343                                 ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN>>10));
1344                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_MIN;
1345         }
1346         else
1347         {
1348                 /* 0 size implies AppHint not set or is set to zero,
1349                  * use default size from driver constant. */
1350                 psDevInfo->ui32RGXFWIfHWPerfBufSize = RGXFW_HWPERF_L1_SIZE_DEFAULT;
1351         }
1352
1353         /* Allocate HWPerf FW L1 buffer */
1354         eError = DevmemFwAllocate(psDevInfo,
1355                                                           psDevInfo->ui32RGXFWIfHWPerfBufSize+RGXFW_HWPERF_L1_PADDING_DEFAULT,
1356                                                           uiMemAllocFlags,
1357                                                           "FirmwareHWPerfBuffer",
1358                                                           &psDevInfo->psRGXFWIfHWPerfBufMemDesc);
1359
1360         if (eError != PVRSRV_OK)
1361         {
1362                 PVR_DPF((PVR_DBG_ERROR, "RGXSetupFirmware: Failed to allocate kernel fw hwperf buffer (%u)",
1363                                  eError));
1364                 goto fail;
1365         }
1366
1367         /* Meta cached flag removed from this allocation as it was found
1368          * FW performance was better without it. */
1369         RGXSetFirmwareAddress(&psRGXFWInit->psHWPerfInfoCtl,
1370                                                   psDevInfo->psRGXFWIfHWPerfBufMemDesc,
1371                                                   0, RFW_FWADDR_NOREF_FLAG);
1372
1373         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfBufMemDesc,
1374                                                                           (IMG_VOID**)&psDevInfo->psRGXFWIfHWPerfBuf);
1375         if (eError != PVRSRV_OK)
1376         {
1377                 PVR_DPF((PVR_DBG_ERROR, "RGXSetupFirmware: Failed to acquire kernel hwperf buffer (%u)",
1378                                  eError));
1379                 goto fail;
1380         }
1381
1382
1383         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1384                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1385                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1386                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1387                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1388                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1389                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1390
1391         /* Allocate buffer to store FW data */
1392         eError = DevmemFwAllocate(psDevInfo,
1393                                                           RGX_META_COREMEM_DATA_SIZE,
1394                                                           uiMemAllocFlags,
1395                                                           "FirmwareCorememDataStore",
1396                                                           &psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
1397
1398         if (eError != PVRSRV_OK)
1399         {
1400                 PVR_DPF((PVR_DBG_ERROR, "RGXSetupFirmware: Failed to allocate coremem data store (%u)",
1401                                  eError));
1402                 goto fail;
1403         }
1404
1405 #if defined(RGX_FEATURE_META_DMA)
1406         RGXSetFirmwareAddress(&sRGXTmpCorememDataStoreFWAddr,
1407                                                   psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1408                                                   0, RFW_FWADDR_NOREF_FLAG);
1409
1410         RGXSetMetaDMAAddress(&psRGXFWInit->sCorememDataStore,
1411                                                  psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1412                                                  &sRGXTmpCorememDataStoreFWAddr,
1413                                                  0);
1414 #else
1415         RGXSetFirmwareAddress(&psRGXFWInit->sCorememDataStore.pbyFWAddr,
1416                                                   psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
1417                                                   0, RFW_FWADDR_NOREF_FLAG);
1418 #endif
1419
1420         /* init HW frame info */
1421         PDUMPCOMMENT("Allocate rgxfw HW info buffer");
1422         eError = DevmemFwAllocate(psDevInfo,
1423                                                         sizeof(RGXFWIF_HWRINFOBUF),
1424                                                         uiMemAllocFlags,
1425                                                         "FirmwareHWInfoBuffer",
1426                                                         &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
1427
1428         if (eError != PVRSRV_OK)
1429         {
1430                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %d bytes for HW info (%u)",
1431                                 (IMG_UINT32)sizeof(RGXFWIF_HWRINFOBUF),
1432                                 eError));
1433                 goto fail;
1434         }
1435
1436         RGXSetFirmwareAddress(&psRGXFWInit->psRGXFWIfHWRInfoBufCtl,
1437                                                 psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
1438                                                 0, RFW_FWADDR_NOREF_FLAG);
1439
1440         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
1441                                                                           (IMG_VOID **)&psDevInfo->psRGXFWIfHWRInfoBuf);
1442         if (eError != PVRSRV_OK)
1443         {
1444                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel tracebuf ctl (%u)",
1445                                 eError));
1446                 goto fail;
1447         }
1448         OSMemSet(psDevInfo->psRGXFWIfHWRInfoBuf, 0, sizeof(RGXFWIF_HWRINFOBUF));
1449
1450         /* init HWPERF data */
1451         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfRIdx = 0;
1452         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfWIdx = 0;
1453         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfWrapCount = 0;
1454         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
1455         psRGXFWInit->ui64HWPerfFilter = ui64HWPerfFilter;
1456         psRGXFWInit->bDisableFilterHWPerfCustomCounter = (ui32ConfigFlags & RGXFWIF_INICFG_HWP_DISABLE_FILTER) ? IMG_TRUE : IMG_FALSE;
1457         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfUt = 0;
1458         psDevInfo->psRGXFWIfTraceBuf->ui32HWPerfDropCount = 0;
1459         psDevInfo->psRGXFWIfTraceBuf->ui32FirstDropOrdinal = 0;
1460         psDevInfo->psRGXFWIfTraceBuf->ui32LastDropOrdinal = 0;
1461         psDevInfo->psRGXFWIfTraceBuf->ui32PowMonEnergy = 0;
1462
1463
1464         /* Initialise the HWPerf module in the Rogue device driver.
1465          * May allocate host buffer if HWPerf enabled at driver load time.
1466          */
1467         eError = RGXHWPerfInit(psDeviceNode, (ui32ConfigFlags & RGXFWIF_INICFG_HWPERF_EN));
1468         PVR_LOGG_IF_ERROR(eError, "RGXHWPerfInit", fail);
1469
1470         /* Set initial log type */
1471         if (ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
1472         {
1473                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1474                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Invalid initial log type (0x%X)",ui32LogType));
1475                 goto fail;
1476         }
1477         psDevInfo->psRGXFWIfTraceBuf->ui32LogType = ui32LogType;
1478
1479         /* Allocate shared buffer for GPU utilisation */
1480         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1481                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1482                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1483                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1484                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1485                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1486                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1487
1488         PDUMPCOMMENT("Allocate shared buffer for GPU utilisation");
1489         eError = DevmemFwAllocate(psDevInfo,
1490                                                         sizeof(RGXFWIF_GPU_UTIL_FWCB),
1491                                                         uiMemAllocFlags,
1492                                                         "FirmwareGPUUtilisationBuffer",
1493                                                         &psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
1494
1495         if (eError != PVRSRV_OK)
1496         {
1497                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for GPU utilisation buffer ctl (%u)",
1498                                 (IMG_UINT32)sizeof(RGXFWIF_GPU_UTIL_FWCB),
1499                                 eError));
1500                 goto fail;
1501         }
1502
1503         RGXSetFirmwareAddress(&psRGXFWInit->psGpuUtilFWCbCtl,
1504                                                 psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc,
1505                                                 0, RFW_FWADDR_NOREF_FLAG);
1506
1507         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc,
1508                                                                           (IMG_VOID **)&psDevInfo->psRGXFWIfGpuUtilFWCb);
1509         if (eError != PVRSRV_OK)
1510         {
1511                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel GPU utilization FW CB ctl (%u)",
1512                                 eError));
1513                 goto fail;
1514         }
1515
1516         /* Initialise GPU utilisation buffer */
1517         psDevInfo->psRGXFWIfGpuUtilFWCb->ui64LastWord =
1518             RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(),RGXFWIF_GPU_UTIL_STATE_IDLE);
1519
1520         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1521                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1522                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1523                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1524                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1525                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1526                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1527
1528         PDUMPCOMMENT("Allocate rgxfw FW runtime configuration (FW)");
1529         eError = DevmemFwAllocate(psDevInfo,
1530                                                         sizeof(RGXFWIF_RUNTIME_CFG),
1531                                                         uiMemAllocFlags,
1532                                                         "FirmwareFWRuntimeCfg",
1533                                                         &psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
1534
1535         if (eError != PVRSRV_OK)
1536         {
1537                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for FW runtime configuration (%u)",
1538                                 (IMG_UINT32)sizeof(RGXFWIF_RUNTIME_CFG),
1539                                 eError));
1540                 goto fail;
1541         }
1542
1543         RGXSetFirmwareAddress(&psRGXFWInit->psRuntimeCfg,
1544                                                 psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
1545                                                 0, RFW_FWADDR_NOREF_FLAG);
1546
1547         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
1548                                                                         (IMG_VOID **)&psDevInfo->psRGXFWIfRuntimeCfg);
1549         if (eError != PVRSRV_OK)
1550         {
1551                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to acquire kernel FW runtime configuration (%u)",
1552                                 eError));
1553                 goto fail;
1554         }
1555
1556 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
1557         PDUMPCOMMENT("Allocate rgxfw register configuration structure");
1558         eError = DevmemFwAllocate(psDevInfo,
1559                                                         sizeof(RGXFWIF_REG_CFG),
1560                                                         uiMemAllocFlags,
1561                                                         "Firmware register configuration structure",
1562                                                         &psDevInfo->psRGXFWIfRegCfgMemDesc);
1563
1564         if (eError != PVRSRV_OK)
1565         {
1566                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate %u bytes for fw register configurations (%u)",
1567                                 (IMG_UINT32)sizeof(RGXFWIF_REG_CFG),
1568                                 eError));
1569                 goto fail;
1570         }
1571
1572         RGXSetFirmwareAddress(&psRGXFWInit->psRegCfg,
1573                                                 psDevInfo->psRGXFWIfRegCfgMemDesc,
1574                                                 0, RFW_FWADDR_NOREF_FLAG);
1575 #endif
1576
1577         uiMemAllocFlags =       PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1578                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1579                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1580                                                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1581                                                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1582                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1583                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1584
1585         PDUMPCOMMENT("Allocate rgxfw hwperfctl structure");
1586         eError = DevmemFwAllocateExportable(psDeviceNode,
1587                                                         ui32HWPerfCountersDataSize,
1588                                                         uiMemAllocFlags,
1589                                                         "Firmware hwperf control structure",
1590                                                         &psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
1591
1592         if (eError != PVRSRV_OK)
1593         {
1594                 PVR_DPF((PVR_DBG_ERROR,"RGXInitHWPerfCounters: Failed to allocate %u bytes for fw hwperf control (%u)",
1595                                 ui32HWPerfCountersDataSize,
1596                                 eError));
1597                 goto fail;
1598         }
1599
1600         eError = DevmemExport(psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
1601                               &psDevInfo->sRGXFWHWPerfCountersExportCookie);
1602         if (eError != PVRSRV_OK)
1603         {
1604                 PVR_DPF((PVR_DBG_ERROR,"Failed to export fw hwperf ctl (%u)",
1605                                 eError));
1606                 goto fail;
1607         }
1608
1609         RGXSetFirmwareAddress(&psRGXFWInit->psHWPerfCtl,
1610                                                 psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
1611                                                 0, 0);
1612
1613         /* Allocate a sync for power management */
1614         eError = SyncPrimContextCreate(IMG_NULL,
1615                                                                         psDevInfo->psDeviceNode,
1616                                                                         &psDevInfo->hSyncPrimContext);
1617         if (eError != PVRSRV_OK)
1618         {
1619                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate sync primitive context with error (%u)", eError));
1620                 goto fail;
1621         }
1622
1623         eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
1624         if (eError != PVRSRV_OK)
1625         {
1626                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate sync primitive with error (%u)", eError));
1627                 goto fail;
1628         }
1629
1630         psRGXFWInit->uiPowerSync = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim);
1631
1632         /* Required info by FW to calculate the ActivePM idle timer latency */
1633         {
1634                 RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
1635                 RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
1636
1637                 psRGXFWInit->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
1638                 psRGXFWInit->ui32ActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
1639
1640                 /* Initialise variable runtime configuration to the system defaults */
1641                 psRuntimeCfg->ui32CoreClockSpeed = psRGXFWInit->ui32InitialCoreClockSpeed;
1642                 psRuntimeCfg->ui32ActivePMLatencyms = psRGXFWInit->ui32ActivePMLatencyms;
1643                 psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
1644         }
1645
1646         /* Setup Fault read register */
1647         eError = RGXSetupFaultReadRegister(psDeviceNode, psRGXFWInit);
1648         if (eError != PVRSRV_OK)
1649         {
1650                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup fault read register"));
1651                 goto fail;
1652         }
1653
1654         /* Apply FIX_HW_BRN_37200 */
1655         eError = RGXHwBrn37200(psDevInfo);
1656         if (eError != PVRSRV_OK)
1657         {
1658                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to apply HWBRN37200"));
1659                 goto fail;
1660         }
1661
1662         /*
1663          * Set up kernel TA CCB.
1664          */
1665         eError = RGXSetupKernelCCB(psDevInfo,
1666                                                            psRGXFWInit,
1667                                                            RGXFWIF_DM_TA, RGXFWIF_KCCB_TA_NUMCMDS_LOG2,
1668                                                            sizeof(RGXFWIF_KCCB_CMD));
1669         if (eError != PVRSRV_OK)
1670         {
1671                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel TA CCB"));
1672                 goto fail;
1673         }
1674
1675         /*
1676          * Set up firmware TA CCB.
1677          */
1678         eError = RGXSetupFirmwareCCB(psDevInfo,
1679                                                            psRGXFWInit,
1680                                                            RGXFWIF_DM_TA, RGXFWIF_FWCCB_TA_NUMCMDS_LOG2,
1681                                                            sizeof(RGXFWIF_FWCCB_CMD));
1682         if (eError != PVRSRV_OK)
1683         {
1684                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware TA CCB"));
1685                 goto fail;
1686         }
1687
1688         /*
1689          * Set up kernel 3D CCB.
1690          */
1691         eError = RGXSetupKernelCCB(psDevInfo,
1692                                                            psRGXFWInit,
1693                                                            RGXFWIF_DM_3D, RGXFWIF_KCCB_3D_NUMCMDS_LOG2,
1694                                                            sizeof(RGXFWIF_KCCB_CMD));
1695         if (eError != PVRSRV_OK)
1696         {
1697                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel 3D CCB"));
1698                 goto fail;
1699         }
1700
1701         /*
1702          * Set up Firmware 3D CCB.
1703          */
1704         eError = RGXSetupFirmwareCCB(psDevInfo,
1705                                                            psRGXFWInit,
1706                                                            RGXFWIF_DM_3D, RGXFWIF_FWCCB_3D_NUMCMDS_LOG2,
1707                                                            sizeof(RGXFWIF_FWCCB_CMD));
1708         if (eError != PVRSRV_OK)
1709         {
1710                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware 3D CCB"));
1711                 goto fail;
1712         }
1713
1714         /*
1715          * Set up kernel 2D CCB.
1716          */
1717         eError = RGXSetupKernelCCB(psDevInfo,
1718                                                            psRGXFWInit,
1719                                                            RGXFWIF_DM_2D, RGXFWIF_KCCB_2D_NUMCMDS_LOG2,
1720                                                            sizeof(RGXFWIF_KCCB_CMD));
1721         if (eError != PVRSRV_OK)
1722         {
1723                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel 2D CCB"));
1724                 goto fail;
1725         }
1726         /*
1727          * Set up Firmware 2D CCB.
1728          */
1729         eError = RGXSetupFirmwareCCB(psDevInfo,
1730                                                            psRGXFWInit,
1731                                                            RGXFWIF_DM_2D, RGXFWIF_FWCCB_2D_NUMCMDS_LOG2,
1732                                                            sizeof(RGXFWIF_FWCCB_CMD));
1733         if (eError != PVRSRV_OK)
1734         {
1735                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware 2D CCB"));
1736                 goto fail;
1737         }
1738
1739         /*
1740          * Set up kernel compute CCB.
1741          */
1742         eError = RGXSetupKernelCCB(psDevInfo,
1743                                                            psRGXFWInit,
1744                                                            RGXFWIF_DM_CDM, RGXFWIF_KCCB_CDM_NUMCMDS_LOG2,
1745                                                            sizeof(RGXFWIF_KCCB_CMD));
1746         if (eError != PVRSRV_OK)
1747         {
1748                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel Compute CCB"));
1749                 goto fail;
1750         }
1751
1752         /*
1753          * Set up Firmware Compute CCB.
1754          */
1755         eError = RGXSetupFirmwareCCB(psDevInfo,
1756                                                            psRGXFWInit,
1757                                                            RGXFWIF_DM_CDM, RGXFWIF_FWCCB_CDM_NUMCMDS_LOG2,
1758                                                            sizeof(RGXFWIF_FWCCB_CMD));
1759         if (eError != PVRSRV_OK)
1760         {
1761                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware Compute CCB"));
1762                 goto fail;
1763         }
1764
1765         /*
1766          * Set up kernel general purpose CCB.
1767          */
1768         eError = RGXSetupKernelCCB(psDevInfo,
1769                                                            psRGXFWInit,
1770                                                            RGXFWIF_DM_GP, RGXFWIF_KCCB_GP_NUMCMDS_LOG2,
1771                                                            sizeof(RGXFWIF_KCCB_CMD));
1772         if (eError != PVRSRV_OK)
1773         {
1774                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel General Purpose CCB"));
1775                 goto fail;
1776         }
1777
1778         /*
1779          * Set up Firmware general purpose CCB.
1780          */
1781         eError = RGXSetupFirmwareCCB(psDevInfo,
1782                                                            psRGXFWInit,
1783                                                            RGXFWIF_DM_GP, RGXFWIF_FWCCB_GP_NUMCMDS_LOG2,
1784                                                            sizeof(RGXFWIF_FWCCB_CMD));
1785         if (eError != PVRSRV_OK)
1786         {
1787                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware General Purpose CCB"));
1788                 goto fail;
1789         }
1790 #if defined(RGX_FEATURE_RAY_TRACING)
1791         /*
1792          * Set up kernel SHG CCB.
1793          */
1794         eError = RGXSetupKernelCCB(psDevInfo,
1795                                                            psRGXFWInit,
1796                                                            RGXFWIF_DM_SHG, RGXFWIF_KCCB_SHG_NUMCMDS_LOG2,
1797                                                            sizeof(RGXFWIF_KCCB_CMD));
1798         if (eError != PVRSRV_OK)
1799         {
1800                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel SHG CCB"));
1801                 goto fail;
1802         }
1803
1804         /*
1805          * Set up Firmware SHG CCB.
1806          */
1807         eError = RGXSetupFirmwareCCB(psDevInfo,
1808                                                            psRGXFWInit,
1809                                                            RGXFWIF_DM_SHG, RGXFWIF_FWCCB_SHG_NUMCMDS_LOG2,
1810                                                            sizeof(RGXFWIF_FWCCB_CMD));
1811         if (eError != PVRSRV_OK)
1812         {
1813                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware SHG CCB"));
1814                 goto fail;
1815         }
1816
1817         /*
1818          * Set up kernel RTU CCB.
1819          */
1820         eError = RGXSetupKernelCCB(psDevInfo,
1821                                                            psRGXFWInit,
1822                                                            RGXFWIF_DM_RTU, RGXFWIF_KCCB_RTU_NUMCMDS_LOG2,
1823                                                            sizeof(RGXFWIF_KCCB_CMD));
1824         if (eError != PVRSRV_OK)
1825         {
1826                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate kernel RTU CCB"));
1827                 goto fail;
1828         }
1829
1830         /*
1831          * Set up Firmware RTU CCB.
1832          */
1833         eError = RGXSetupFirmwareCCB(psDevInfo,
1834                                                            psRGXFWInit,
1835                                                            RGXFWIF_DM_RTU, RGXFWIF_FWCCB_RTU_NUMCMDS_LOG2,
1836                                                            sizeof(RGXFWIF_FWCCB_CMD));
1837         if (eError != PVRSRV_OK)
1838         {
1839                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate Firmware SHG CCB"));
1840                 goto fail;
1841         }
1842 #endif
1843
1844         /* Setup Signature and Checksum Buffers for TA and 3D */
1845         eError = RGXFWSetupSignatureChecks(psDevInfo,
1846                                            &psDevInfo->psRGXFWSigTAChecksMemDesc,
1847                                            ui32SignatureChecksBufSize,
1848                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_TA],
1849                                            "TA");
1850         if (eError != PVRSRV_OK)
1851         {
1852                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup TA signature checks"));
1853                 goto fail;
1854         }
1855         psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
1856
1857         eError = RGXFWSetupSignatureChecks(psDevInfo,
1858                                            &psDevInfo->psRGXFWSig3DChecksMemDesc,
1859                                            ui32SignatureChecksBufSize,
1860                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_3D],
1861                                            "3D");
1862         if (eError != PVRSRV_OK)
1863         {
1864                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup 3D signature checks"));
1865                 goto fail;
1866         }
1867         psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
1868
1869 #if defined(RGX_FEATURE_RAY_TRACING)
1870         eError = RGXFWSetupSignatureChecks(psDevInfo,
1871                                            &psDevInfo->psRGXFWSigRTChecksMemDesc,
1872                                            ui32SignatureChecksBufSize,
1873                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_RTU],
1874                                            "RTU");
1875         if (eError != PVRSRV_OK)
1876         {
1877                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup RTU signature checks"));
1878                 goto fail;
1879         }
1880         psDevInfo->ui32SigRTChecksSize = ui32SignatureChecksBufSize;
1881
1882         eError = RGXFWSetupSignatureChecks(psDevInfo,
1883                                            &psDevInfo->psRGXFWSigSHChecksMemDesc,
1884                                            ui32SignatureChecksBufSize,
1885                                            &psRGXFWInit->asSigBufCtl[RGXFWIF_DM_SHG],
1886                                            "SHG");
1887         if (eError != PVRSRV_OK)
1888         {
1889                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup SHG signature checks"));
1890                 goto fail;
1891         }
1892         psDevInfo->ui32SigSHChecksSize = ui32SignatureChecksBufSize;
1893 #endif
1894
1895 #if defined(RGXFW_ALIGNCHECKS)
1896         eError = RGXFWSetupAlignChecks(psDevInfo,
1897                                                                 &psRGXFWInit->paui32AlignChecks,
1898                                                                 pui32RGXFWAlignChecks,
1899                                                                 ui32RGXFWAlignChecksSize);
1900         if (eError != PVRSRV_OK)
1901         {
1902                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to setup alignment checks"));
1903                 goto fail;
1904         }
1905 #endif
1906
1907         /* Fill the remaining bits of fw the init data */
1908         psRGXFWInit->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
1909         psRGXFWInit->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
1910         psRGXFWInit->sDPXControlStreamBase.uiAddr = RGX_DOPPLER_HEAP_BASE;
1911         psRGXFWInit->sResultDumpBase.uiAddr = RGX_DOPPLER_OVERFLOW_HEAP_BASE;
1912         psRGXFWInit->sRTUHeapBase.uiAddr = RGX_DOPPLER_HEAP_BASE;
1913
1914         /* RD Power Island */
1915         {
1916                 RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
1917                 IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
1918                 IMG_BOOL bEnableRDPowIsland = ((eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
1919                                                 (eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
1920
1921                 ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
1922         }
1923
1924         psRGXFWInit->ui32ConfigFlags = ui32ConfigFlags & RGXFWIF_INICFG_ALL;
1925         psRGXFWInit->ui32FilterFlags = ui32FilterFlags;
1926 #if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
1927         psRGXFWInit->ui32JonesDisableMask = ui32JonesDisableMask;
1928 #endif
1929         psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_SRVCFG_DISABLE_PDP_EN)
1930                         ? IMG_FALSE : IMG_TRUE;
1931         psRGXFWInit->ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
1932
1933 #if defined(RGX_FEATURE_SLC_VIVT)
1934         eError = _AllocateSLC3Fence(psDevInfo, psRGXFWInit);
1935         if (eError != PVRSRV_OK)
1936         {
1937                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to allocate memory for SLC3Fence"));
1938                 goto fail;
1939         }
1940 #endif
1941
1942         /* Timestamps */
1943         uiMemAllocFlags =
1944                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1945                 PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
1946                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1947                 PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1948                 PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
1949                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1950                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1951
1952         /*
1953           the timer query arrays
1954         */
1955         PDUMPCOMMENT("Allocate timer query arrays (FW)");
1956         eError = DevmemFwAllocate(psDevInfo,
1957                                   sizeof(RGXFWIF_TIMESTAMP) * RGX_MAX_TIMER_QUERIES,
1958                                   uiMemAllocFlags,
1959                                   "Start times array",
1960                                   & psDevInfo->psStartTimeMemDesc);
1961         if (eError != PVRSRV_OK)
1962         {
1963                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map start times array"));
1964                 goto fail;
1965         }
1966
1967         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
1968                                           (IMG_VOID **)& psDevInfo->pasStartTimeById);
1969
1970         if (eError != PVRSRV_OK)
1971         {
1972                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map start times array"));
1973                 goto fail;
1974         }
1975
1976         eError = DevmemFwAllocate(psDevInfo,
1977                                   sizeof(RGXFWIF_TIMESTAMP) * RGX_MAX_TIMER_QUERIES,
1978                                   uiMemAllocFlags,
1979                                   "End times array",
1980                                   & psDevInfo->psEndTimeMemDesc);
1981         if (eError != PVRSRV_OK)
1982         {
1983                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map end times array"));
1984                 goto fail;
1985         }
1986
1987         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
1988                                           (IMG_VOID **)& psDevInfo->pasEndTimeById);
1989
1990         if (eError != PVRSRV_OK)
1991         {
1992                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map end times array"));
1993                 goto fail;
1994         }
1995
1996         eError = DevmemFwAllocate(psDevInfo,
1997                                   sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
1998                                   uiMemAllocFlags,
1999                                   "Completed ops array",
2000                                   & psDevInfo->psCompletedMemDesc);
2001         if (eError != PVRSRV_OK)
2002         {
2003                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to completed ops array"));
2004                 goto fail;
2005         }
2006
2007         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
2008                                           (IMG_VOID **)& psDevInfo->pui32CompletedById);
2009
2010         if (eError != PVRSRV_OK)
2011         {
2012                 PVR_DPF((PVR_DBG_ERROR,"RGXSetupFirmware: Failed to map completed ops array"));
2013                 goto fail;
2014         }
2015
2016                 /* Initialize FW started flag */
2017         psRGXFWInit->bFirmwareStarted = IMG_FALSE;
2018
2019         /* Initialise the compatibility check data */
2020         RGXFWIF_COMPCHECKS_BVNC_INIT(psRGXFWInit->sRGXCompChecks.sFWBVNC);
2021         RGXFWIF_COMPCHECKS_BVNC_INIT(psRGXFWInit->sRGXCompChecks.sHWBVNC);
2022
2023         {
2024                 /* Below line is to make sure (compilation time check) that
2025                                 RGX_BVNC_KM_V_ST fits into RGXFWIF_COMPCHECKS_BVNC structure */
2026                 IMG_CHAR _tmp_[RGXFWIF_COMPCHECKS_BVNC_V_LEN_MAX] = RGX_BVNC_KM_V_ST;
2027                 _tmp_[0] = '\0';
2028         }
2029
2030         PDUMPCOMMENT("Dump RGXFW Init data");
2031         if (!bEnableSignatureChecks)
2032         {
2033 #if defined(PDUMP)
2034                 PDUMPCOMMENT("(to enable rgxfw signatures place the following line after the RTCONF line)");
2035                 DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfInitMemDesc,
2036                                                         offsetof(RGXFWIF_INIT, asSigBufCtl),
2037                                                         sizeof(RGXFWIF_SIGBUF_CTL)*RGXFWIF_DM_MAX,
2038                                                         PDUMP_FLAGS_CONTINUOUS);
2039 #endif
2040                 psRGXFWInit->asSigBufCtl[RGXFWIF_DM_3D].psBuffer.ui32Addr = 0x0;
2041                 psRGXFWInit->asSigBufCtl[RGXFWIF_DM_TA].psBuffer.ui32Addr = 0x0;
2042         }
2043
2044         for (dm = 0; dm < RGXFWIF_DM_MAX; dm++)
2045         {
2046                 psDevInfo->psRGXFWIfTraceBuf->aui16HwrDmLockedUpCount[dm] = 0;
2047                 psDevInfo->psRGXFWIfTraceBuf->aui16HwrDmOverranCount[dm] = 0;
2048                 psDevInfo->psRGXFWIfTraceBuf->aui16HwrDmRecoveredCount[dm] = 0;
2049                 psDevInfo->psRGXFWIfTraceBuf->aui16HwrDmFalseDetectCount[dm] = 0;
2050                 psDevInfo->psRGXFWIfTraceBuf->apsHwrDmFWCommonContext[dm].ui32Addr = 0;
2051         }
2052
2053         /*
2054          * BIF Tiling configuration
2055          */
2056
2057         psRGXFWInit->sBifTilingCfg[0].uiBase = RGX_BIF_TILING_HEAP_1_BASE;
2058         psRGXFWInit->sBifTilingCfg[0].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2059         psRGXFWInit->sBifTilingCfg[0].uiXStride = pui32BIFTilingXStrides[0];
2060         psRGXFWInit->sBifTilingCfg[1].uiBase = RGX_BIF_TILING_HEAP_2_BASE;
2061         psRGXFWInit->sBifTilingCfg[1].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2062         psRGXFWInit->sBifTilingCfg[1].uiXStride = pui32BIFTilingXStrides[1];
2063         psRGXFWInit->sBifTilingCfg[2].uiBase = RGX_BIF_TILING_HEAP_3_BASE;
2064         psRGXFWInit->sBifTilingCfg[2].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2065         psRGXFWInit->sBifTilingCfg[2].uiXStride = pui32BIFTilingXStrides[2];
2066         psRGXFWInit->sBifTilingCfg[3].uiBase = RGX_BIF_TILING_HEAP_4_BASE;
2067         psRGXFWInit->sBifTilingCfg[3].uiLen = RGX_BIF_TILING_HEAP_SIZE;
2068         psRGXFWInit->sBifTilingCfg[3].uiXStride = pui32BIFTilingXStrides[3];
2069
2070 #if defined(PDUMP)
2071         PDUMPCOMMENT("Dump rgxfw hwperfctl structure");
2072         DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
2073                                 0,
2074                                                         ui32HWPerfCountersDataSize,
2075                                 PDUMP_FLAGS_CONTINUOUS);
2076         PDUMPCOMMENT("Dump rgxfw HW Perf Info structure");
2077         DevmemPDumpLoadMem (psDevInfo->psRGXFWIfHWPerfBufMemDesc,
2078                                                 0,
2079                                                 psDevInfo->ui32RGXFWIfHWPerfBufSize,
2080                                                 PDUMP_FLAGS_CONTINUOUS);
2081         PDUMPCOMMENT("Dump rgxfw trace structure");
2082         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
2083                                                 0,
2084                                                 sizeof(RGXFWIF_TRACEBUF),
2085                                                 PDUMP_FLAGS_CONTINUOUS);
2086 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2087         PDUMPCOMMENT("Dump rgxfw register configuration buffer");
2088         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfRegCfgMemDesc,
2089                                                 0,
2090                                                 sizeof(RGXFWIF_REG_CFG),
2091                                                 PDUMP_FLAGS_CONTINUOUS);
2092 #endif
2093         PDUMPCOMMENT("Dump rgxfw init structure");
2094         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfInitMemDesc,
2095                                                 0,
2096                                                 sizeof(RGXFWIF_INIT),
2097                                                 PDUMP_FLAGS_CONTINUOUS);
2098
2099         PDUMPCOMMENT("Dump rgxfw coremem data store");
2100         DevmemPDumpLoadMem(     psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
2101                                                 0,
2102                                                 RGX_META_COREMEM_DATA_SIZE,
2103                                                 PDUMP_FLAGS_CONTINUOUS);
2104
2105         PDUMPCOMMENT("RTCONF: run-time configuration");
2106
2107
2108         /* Dump the config options so they can be edited.
2109          *
2110          * FIXME: Need new DevmemPDumpWRW API which writes a WRW to load ui32ConfigFlags
2111          */
2112         PDUMPCOMMENT("(Set the FW config options here)");
2113         PDUMPCOMMENT("( Ctx Switch TA Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_TA_EN);
2114         PDUMPCOMMENT("( Ctx Switch 3D Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_3D_EN);
2115         PDUMPCOMMENT("( Ctx Switch CDM Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_CDM_EN);
2116         PDUMPCOMMENT("( Ctx Switch Rand mode: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
2117         PDUMPCOMMENT("( Ctx Switch Soft Reset Enable: 0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
2118         PDUMPCOMMENT("( Reserved (do not set): 0x%08x)", RGXFWIF_INICFG_RSVD);
2119         PDUMPCOMMENT("( Rascal+Dust Power Island: 0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
2120         PDUMPCOMMENT("( Enable HWPerf: 0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
2121         PDUMPCOMMENT("( Enable HWR: 0x%08x)", RGXFWIF_INICFG_HWR_EN);
2122         PDUMPCOMMENT("( Check MList: 0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
2123         PDUMPCOMMENT("( Disable Auto Clock Gating: 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
2124         PDUMPCOMMENT("( Enable HWPerf Polling Perf Counter: 0x%08x)", RGXFWIF_INICFG_POLL_COUNTERS_EN);
2125 #if defined(RGX_FEATURE_VDM_OBJECT_LEVEL_LLS)
2126         PDUMPCOMMENT("( Ctx Switch Object mode Index: 0x%08x)", RGXFWIF_INICFG_VDM_CTX_STORE_MODE_INDEX);
2127         PDUMPCOMMENT("( Ctx Switch Object mode Instance: 0x%08x)", RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INSTANCE);
2128         PDUMPCOMMENT("( Ctx Switch Object mode List: 0x%08x)", RGXFWIF_INICFG_VDM_CTX_STORE_MODE_LIST);
2129 #endif
2130         PDUMPCOMMENT("( Enable SHG Bypass mode: 0x%08x)", RGXFWIF_INICFG_SHG_BYPASS_EN);
2131         PDUMPCOMMENT("( Enable RTU Bypass mode: 0x%08x)", RGXFWIF_INICFG_RTU_BYPASS_EN);
2132         PDUMPCOMMENT("( Enable register configuration: 0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
2133         PDUMPCOMMENT("( Assert on TA Out-of-Memory: 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
2134         PDUMPCOMMENT("( Disable HWPerf custom counter filter: 0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
2135         PDUMPCOMMENT("( Enable HWPerf custom performance timer: 0x%08x)", RGXFWIF_INICFG_CUSTOM_PERF_TIMER_EN);
2136         PDUMPCOMMENT("( Enable CDM Killing Rand mode: 0x%08x)", RGXFWIF_INICFG_CDM_KILL_MODE_RAND_EN);
2137         PDUMPCOMMENT("( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
2138
2139         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2140                                                         offsetof(RGXFWIF_INIT, ui32ConfigFlags),
2141                                                         psRGXFWInit->ui32ConfigFlags,
2142                                                         PDUMP_FLAGS_CONTINUOUS);
2143
2144         /* default: no filter */
2145         psRGXFWInit->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
2146         psRGXFWInit->sPIDFilter.asItems[0].uiPID = 0;
2147
2148         PDUMPCOMMENT("( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
2149                                                         RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
2150                                                         RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
2151
2152         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2153                                                         offsetof(RGXFWIF_INIT, sPIDFilter.eMode),
2154                                                         psRGXFWInit->sPIDFilter.eMode,
2155                                                         PDUMP_FLAGS_CONTINUOUS);
2156
2157         PDUMPCOMMENT("( PID filter PID/OSID list (Up to %u entries. Terminate with a zero PID))",
2158                                                                         RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
2159         {
2160                 IMG_UINT32 i;
2161
2162                 /* generate a few WRWs in the pdump stream as an example */
2163                 for(i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
2164                 {
2165                         PDUMPCOMMENT("(PID and OSID pair %u)", i);
2166
2167                         PDUMPCOMMENT("(PID)");
2168                         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2169                                                 offsetof(RGXFWIF_INIT, sPIDFilter.asItems[i].uiPID),
2170                                                 0,
2171                                                 PDUMP_FLAGS_CONTINUOUS);
2172
2173                         PDUMPCOMMENT("(OSID)");
2174                         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfInitMemDesc,
2175                                                 offsetof(RGXFWIF_INIT, sPIDFilter.asItems[i].ui32OSID),
2176                                                 0,
2177                                                 PDUMP_FLAGS_CONTINUOUS);
2178                 }
2179         }
2180
2181         /*
2182          * Dump the log config so it can be edited.
2183          */
2184         PDUMPCOMMENT("(Set the log config here)");
2185         PDUMPCOMMENT("( Log Type: set bit 0 for TRACE, reset for TBI)");
2186         PDUMPCOMMENT("( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
2187         PDUMPCOMMENT("( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
2188         PDUMPCOMMENT("( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
2189         PDUMPCOMMENT("( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
2190         PDUMPCOMMENT("( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
2191         PDUMPCOMMENT("( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
2192         PDUMPCOMMENT("( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
2193         PDUMPCOMMENT("( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
2194         PDUMPCOMMENT("( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
2195         PDUMPCOMMENT("( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
2196         PDUMPCOMMENT("( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
2197 #if defined(RGX_FEATURE_RAY_TRACING)
2198         PDUMPCOMMENT("( RPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RPM);
2199 #endif
2200 #if defined(RGX_FEATURE_META_DMA)
2201         PDUMPCOMMENT("( DMA Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DMA);
2202 #endif
2203         PDUMPCOMMENT("( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
2204         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
2205                                                         offsetof(RGXFWIF_TRACEBUF, ui32LogType),
2206                                                         psDevInfo->psRGXFWIfTraceBuf->ui32LogType,
2207                                                         PDUMP_FLAGS_CONTINUOUS);
2208
2209         PDUMPCOMMENT("Set the HWPerf Filter config here");
2210         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfInitMemDesc,
2211                                                 offsetof(RGXFWIF_INIT, ui64HWPerfFilter),
2212                                                 psRGXFWInit->ui64HWPerfFilter,
2213                                                 PDUMP_FLAGS_CONTINUOUS);
2214
2215 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2216         PDUMPCOMMENT("(Number of registers configurations in sidekick)");
2217         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2218                                                         offsetof(RGXFWIF_REG_CFG, ui32NumRegsSidekick),
2219                                                         0,
2220                                                         PDUMP_FLAGS_CONTINUOUS);
2221         PDUMPCOMMENT("(Number of registers configurations in rascal/dust)");
2222         DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
2223                                                         offsetof(RGXFWIF_REG_CFG, ui32NumRegsRascalDust),
2224                                                         0,
2225                                                         PDUMP_FLAGS_CONTINUOUS);
2226         PDUMPCOMMENT("(Set registers here, address, value)");
2227         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
2228                                                         offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
2229                                                         0,
2230                                                         PDUMP_FLAGS_CONTINUOUS);
2231         DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
2232                                                         offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
2233                                                         0,
2234                                                         PDUMP_FLAGS_CONTINUOUS);
2235 #endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
2236 #endif
2237
2238         /* We don't need access to the fw init data structure anymore */
2239         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
2240         psRGXFWInit = IMG_NULL;
2241
2242         psDevInfo->bFirmwareInitialised = IMG_TRUE;
2243
2244         return PVRSRV_OK;
2245
2246 fail:
2247         if (psDevInfo->psRGXFWIfInitMemDesc != IMG_NULL && psRGXFWInit != IMG_NULL)
2248         {
2249                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
2250         }
2251         RGXFreeFirmware(psDevInfo);
2252         PVR_ASSERT(eError != PVRSRV_OK);
2253         return eError;
2254 }
2255
2256 /*!
2257 *******************************************************************************
2258
2259  @Function      RGXFreeFirmware
2260
2261  @Description
2262
2263  Frees all the firmware-related allocations
2264
2265  @Input psDevInfo
2266
2267  @Return PVRSRV_ERROR
2268
2269 ******************************************************************************/
2270 IMG_VOID RGXFreeFirmware(PVRSRV_RGXDEV_INFO     *psDevInfo)
2271 {
2272         RGXFWIF_DM      eCCBType;
2273
2274         psDevInfo->bFirmwareInitialised = IMG_FALSE;
2275
2276         for (eCCBType = 0; eCCBType < RGXFWIF_DM_MAX; eCCBType++)
2277         {
2278                 RGXFreeKernelCCB(psDevInfo, eCCBType);
2279                 RGXFreeFirmwareCCB(psDevInfo, eCCBType);
2280         }
2281
2282 #if defined(RGXFW_ALIGNCHECKS)
2283         if (psDevInfo->psRGXFWAlignChecksMemDesc)
2284         {
2285                 RGXFWFreeAlignChecks(psDevInfo);
2286         }
2287 #endif
2288
2289         if (psDevInfo->psRGXFWSigTAChecksMemDesc)
2290         {
2291                 DevmemFwFree(psDevInfo->psRGXFWSigTAChecksMemDesc);
2292                 psDevInfo->psRGXFWSigTAChecksMemDesc = IMG_NULL;
2293         }
2294
2295         if (psDevInfo->psRGXFWSig3DChecksMemDesc)
2296         {
2297                 DevmemFwFree(psDevInfo->psRGXFWSig3DChecksMemDesc);
2298                 psDevInfo->psRGXFWSig3DChecksMemDesc = IMG_NULL;
2299         }
2300
2301 #if defined(RGX_FEATURE_RAY_TRACING)
2302         if (psDevInfo->psRGXFWSigRTChecksMemDesc)
2303         {
2304                 DevmemFwFree(psDevInfo->psRGXFWSigRTChecksMemDesc);
2305                 psDevInfo->psRGXFWSigRTChecksMemDesc = IMG_NULL;
2306         }
2307
2308         if (psDevInfo->psRGXFWSigSHChecksMemDesc)
2309         {
2310                 DevmemFwFree(psDevInfo->psRGXFWSigSHChecksMemDesc);
2311                 psDevInfo->psRGXFWSigSHChecksMemDesc = IMG_NULL;
2312         }
2313 #endif
2314
2315 #if defined(FIX_HW_BRN_37200)
2316         if (psDevInfo->psRGXFWHWBRN37200MemDesc)
2317         {
2318                 DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWHWBRN37200MemDesc);
2319                 DevmemFree(psDevInfo->psRGXFWHWBRN37200MemDesc);
2320                 psDevInfo->psRGXFWHWBRN37200MemDesc = IMG_NULL;
2321         }
2322 #endif
2323
2324         RGXSetupFaultReadRegisterRollback(psDevInfo);
2325
2326         if (psDevInfo->psPowSyncPrim != IMG_NULL)
2327         {
2328                 SyncPrimFree(psDevInfo->psPowSyncPrim);
2329                 psDevInfo->psPowSyncPrim = IMG_NULL;
2330         }
2331
2332         if (psDevInfo->hSyncPrimContext != 0)
2333         {
2334                 SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
2335                 psDevInfo->hSyncPrimContext = 0;
2336         }
2337
2338         if (psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc)
2339         {
2340                 if (psDevInfo->psRGXFWIfGpuUtilFWCb != IMG_NULL)
2341                 {
2342                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
2343                         psDevInfo->psRGXFWIfGpuUtilFWCb = IMG_NULL;
2344                 }
2345                 DevmemFwFree(psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc);
2346                 psDevInfo->psRGXFWIfGpuUtilFWCbCtlMemDesc = IMG_NULL;
2347         }
2348
2349         if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
2350         {
2351                 if (psDevInfo->psRGXFWIfRuntimeCfg != IMG_NULL)
2352                 {
2353                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
2354                         psDevInfo->psRGXFWIfRuntimeCfg = IMG_NULL;
2355                 }
2356                 DevmemFwFree(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
2357                 psDevInfo->psRGXFWIfRuntimeCfgMemDesc = IMG_NULL;
2358         }
2359
2360         if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
2361         {
2362                 if (psDevInfo->psRGXFWIfHWRInfoBuf != IMG_NULL)
2363                 {
2364                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
2365                         psDevInfo->psRGXFWIfHWRInfoBuf = IMG_NULL;
2366                 }
2367                 DevmemFwFree(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
2368                 psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = IMG_NULL;
2369         }
2370
2371         RGXHWPerfDeinit();
2372
2373         if (psDevInfo->psRGXFWIfHWPerfBufMemDesc)
2374         {
2375                 if (psDevInfo->psRGXFWIfHWPerfBuf != IMG_NULL)
2376                 {
2377                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
2378                         psDevInfo->psRGXFWIfHWPerfBuf = IMG_NULL;
2379                 }
2380                 DevmemFwFree(psDevInfo->psRGXFWIfHWPerfBufMemDesc);
2381                 psDevInfo->psRGXFWIfHWPerfBufMemDesc = IMG_NULL;
2382         }
2383
2384         if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
2385         {
2386                 DevmemFwFree(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
2387                 psDevInfo->psRGXFWIfCorememDataStoreMemDesc = IMG_NULL;
2388         }
2389
2390         if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
2391         {
2392                 if (psDevInfo->psRGXFWIfTraceBuf != IMG_NULL)
2393                 {
2394                         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
2395                         psDevInfo->psRGXFWIfTraceBuf = IMG_NULL;
2396                 }
2397                 DevmemFwFree(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
2398                 psDevInfo->psRGXFWIfTraceBufCtlMemDesc = IMG_NULL;
2399         }
2400 #if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
2401         if (psDevInfo->psRGXFWIfRegCfgMemDesc)
2402         {
2403                 DevmemFwFree(psDevInfo->psRGXFWIfRegCfgMemDesc);
2404                 psDevInfo->psRGXFWIfRegCfgMemDesc = IMG_NULL;
2405         }
2406 #endif
2407         if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
2408         {
2409                 RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
2410                 if (DevmemIsValidExportCookie(&psDevInfo->sRGXFWHWPerfCountersExportCookie))
2411                 {
2412                         /* if the export cookie is valid, the init sequence failed */
2413                         PVR_DPF((PVR_DBG_ERROR, "RGXFreeFirmware: FW HWPerf Export cookie"
2414                                  "still valid (should have been unexported at init time)"));
2415                         DevmemUnexport(psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
2416                                        &psDevInfo->sRGXFWHWPerfCountersExportCookie);
2417                 }
2418                 DevmemFwFree(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
2419                 psDevInfo->psRGXFWIfHWPerfCountersMemDesc = IMG_NULL;
2420         }
2421 #if defined(RGX_FEATURE_SLC_VIVT)
2422         _FreeSLC3Fence(psDevInfo);
2423 #endif
2424
2425         if (psDevInfo->psRGXFWIfInitMemDesc)
2426         {
2427                 DevmemFwFree(psDevInfo->psRGXFWIfInitMemDesc);
2428                 psDevInfo->psRGXFWIfInitMemDesc = IMG_NULL;
2429         }
2430
2431         if (psDevInfo->psCompletedMemDesc)
2432         {
2433                 if (psDevInfo->pui32CompletedById)
2434                 {
2435                         DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
2436                         psDevInfo->pui32CompletedById = IMG_NULL;
2437                 }
2438                 DevmemFwFree(psDevInfo->psCompletedMemDesc);
2439                 psDevInfo->psCompletedMemDesc = IMG_NULL;
2440         }
2441         if (psDevInfo->psEndTimeMemDesc)
2442         {
2443                 if (psDevInfo->pasEndTimeById)
2444                 {
2445                         DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
2446                         psDevInfo->pasEndTimeById = IMG_NULL;
2447                 }
2448
2449                 DevmemFwFree(psDevInfo->psEndTimeMemDesc);
2450                 psDevInfo->psEndTimeMemDesc = IMG_NULL;
2451         }
2452         if (psDevInfo->psStartTimeMemDesc)
2453         {
2454                 if (psDevInfo->pasStartTimeById)
2455                 {
2456                         DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
2457                         psDevInfo->pasStartTimeById = IMG_NULL;
2458                 }
2459
2460                 DevmemFwFree(psDevInfo->psStartTimeMemDesc);
2461                 psDevInfo->psStartTimeMemDesc = IMG_NULL;
2462         }
2463 }
2464
2465
2466 /******************************************************************************
2467  FUNCTION       : RGXStartFirmware
2468
2469  PURPOSE        : Attempts to obtain a slot in the Kernel CCB
2470
2471  PARAMETERS     : psDevInfo
2472
2473  RETURNS        : PVRSRV_ERROR
2474 ******************************************************************************/
2475 PVRSRV_ERROR RGXStartFirmware(PVRSRV_RGXDEV_INFO        *psDevInfo)
2476 {
2477         PVRSRV_ERROR    eError = PVRSRV_OK;
2478
2479         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Rogue Firmware Slave boot Start");
2480         /*
2481          * Run init script.
2482          */
2483         eError = RGXRunScript(psDevInfo, psDevInfo->psScripts->asInitCommands, RGX_MAX_INIT_COMMANDS, PDUMP_FLAGS_CONTINUOUS, IMG_NULL);
2484         if (eError != PVRSRV_OK)
2485         {
2486                 PVR_DPF((PVR_DBG_ERROR,"RGXStart: RGXRunScript failed (%d)", eError));
2487                 return eError;
2488         }
2489         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Rogue Firmware startup complete\n");
2490
2491         return eError;
2492 }
2493
2494
2495 /******************************************************************************
2496  FUNCTION       : RGXAcquireKernelCCBSlot
2497
2498  PURPOSE        : Attempts to obtain a slot in the Kernel CCB
2499
2500  PARAMETERS     : psCCB - the CCB
2501                         : Address of space if available, IMG_NULL otherwise
2502
2503  RETURNS        : PVRSRV_ERROR
2504 ******************************************************************************/
2505 static PVRSRV_ERROR RGXAcquireKernelCCBSlot(DEVMEM_MEMDESC *psKCCBCtrlMemDesc,
2506                                                                                         RGXFWIF_CCB_CTL *psKCCBCtl,
2507                                                                                         IMG_UINT32                      *pui32Offset)
2508 {
2509         IMG_UINT32      ui32OldWriteOffset, ui32NextWriteOffset;
2510
2511         ui32OldWriteOffset = psKCCBCtl->ui32WriteOffset;
2512         ui32NextWriteOffset = (ui32OldWriteOffset + 1) & psKCCBCtl->ui32WrapMask;
2513
2514         /* Note: The MTS can queue up to 255 kicks (254 pending kicks and 1 executing kick)
2515          * Hence the kernel CCB should not queue more 254 commands
2516          */
2517         PVR_ASSERT(psKCCBCtl->ui32WrapMask < 255);
2518
2519 #if defined(PDUMP)
2520         /* Wait for sufficient CCB space to become available */
2521         PDUMPCOMMENTWITHFLAGS(0, "Wait for kCCB woff=%u", ui32NextWriteOffset);
2522         DevmemPDumpCBP(psKCCBCtrlMemDesc,
2523                        offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
2524                        ui32NextWriteOffset,
2525                        1,
2526                        (psKCCBCtl->ui32WrapMask + 1));
2527 #endif
2528
2529         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
2530         {
2531
2532                 if (ui32NextWriteOffset != psKCCBCtl->ui32ReadOffset)
2533                 {
2534                         *pui32Offset = ui32NextWriteOffset;
2535                         return PVRSRV_OK;
2536                 }
2537                 {
2538                         /*
2539                          * The following sanity check doesn't impact performance,
2540                          * since the CPU has to wait for the GPU anyway (full kernel CCB).
2541                          */
2542                         if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
2543                         {
2544                                 return PVRSRV_ERROR_KERNEL_CCB_FULL;
2545                         }
2546                 }
2547
2548                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
2549         } END_LOOP_UNTIL_TIMEOUT();
2550
2551         /* Time out on waiting for CCB space */
2552         return PVRSRV_ERROR_KERNEL_CCB_FULL;
2553 }
2554
2555
2556 PVRSRV_ERROR RGXSendCommandWithPowLock(PVRSRV_RGXDEV_INFO       *psDevInfo,
2557                                                                                  RGXFWIF_DM                     eKCCBType,
2558                                                                                  RGXFWIF_KCCB_CMD       *psKCCBCmd,
2559                                                                                  IMG_UINT32                     ui32CmdSize,
2560                                                                                  IMG_BOOL                       bPDumpContinuous)
2561 {
2562         PVRSRV_ERROR            eError;
2563         PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
2564
2565         /* Ensure Rogue is powered up before kicking MTS */
2566         eError = PVRSRVPowerLock();
2567
2568         if (eError != PVRSRV_OK)
2569         {
2570                 PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandWithPowLock: failed to acquire powerlock (%s)",
2571                                         PVRSRVGetErrorStringKM(eError)));
2572
2573                 goto _PVRSRVPowerLock_Exit;
2574         }
2575
2576         PDUMPPOWCMDSTART();
2577         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode->sDevId.ui32DeviceIndex,
2578                                                                                  PVRSRV_DEV_POWER_STATE_ON,
2579                                                                                  IMG_FALSE);
2580         PDUMPPOWCMDEND();
2581
2582         if (eError != PVRSRV_OK)
2583         {
2584                 PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandWithPowLock: failed to transition Rogue to ON (%s)",
2585                                         PVRSRVGetErrorStringKM(eError)));
2586
2587                 goto _PVRSRVSetDevicePowerStateKM_Exit;
2588         }
2589
2590         eError = RGXSendCommandRaw(psDevInfo, eKCCBType,  psKCCBCmd, ui32CmdSize, bPDumpContinuous?PDUMP_FLAGS_CONTINUOUS:0);
2591         if (eError != PVRSRV_OK)
2592         {
2593                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandWithPowLock: failed to schedule command (%s)",
2594                                         PVRSRVGetErrorStringKM(eError)));
2595 #if defined(DEBUG)
2596                 /* PVRSRVDebugRequest must be called without powerlock */
2597                 PVRSRVPowerUnlock();
2598                 PVRSRVDebugRequest(DEBUG_REQUEST_VERBOSITY_MAX, IMG_NULL);
2599                 goto _PVRSRVPowerLock_Exit;
2600 #endif
2601         }
2602
2603 _PVRSRVSetDevicePowerStateKM_Exit:
2604         PVRSRVPowerUnlock();
2605
2606 _PVRSRVPowerLock_Exit:
2607         return eError;
2608 }
2609
2610 PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO       *psDevInfo,
2611                                                                  RGXFWIF_DM                     eKCCBType,
2612                                                                  RGXFWIF_KCCB_CMD       *psKCCBCmd,
2613                                                                  IMG_UINT32                     ui32CmdSize,
2614                                                                  PDUMP_FLAGS_T          uiPdumpFlags)
2615 {
2616         PVRSRV_ERROR            eError;
2617         RGXFWIF_CCB_CTL         *psKCCBCtl = psDevInfo->apsKernelCCBCtl[eKCCBType];
2618         IMG_UINT8                       *pui8KCCB = psDevInfo->apsKernelCCB[eKCCBType];
2619         IMG_UINT32                      ui32NewWriteOffset;
2620         IMG_UINT32                      ui32OldWriteOffset = psKCCBCtl->ui32WriteOffset;
2621 #if !defined(PDUMP)
2622         PVR_UNREFERENCED_PARAMETER(uiPdumpFlags);
2623 #endif
2624 #if defined(PDUMP)
2625         IMG_BOOL bIsInCaptureRange;
2626         IMG_BOOL bPdumpEnabled;
2627         IMG_BOOL bPDumpContinuous = (uiPdumpFlags & PDUMP_FLAGS_CONTINUOUS) != 0;
2628         IMG_BOOL bPDumpPowTrans = PDUMPPOWCMDINTRANS();
2629
2630         PDumpIsCaptureFrameKM(&bIsInCaptureRange);
2631         bPdumpEnabled = (bIsInCaptureRange || bPDumpContinuous) && !bPDumpPowTrans;
2632
2633         /* in capture range */
2634         if (bPdumpEnabled)
2635         {
2636                 if (!psDevInfo->abDumpedKCCBCtlAlready[eKCCBType])
2637                 {
2638                         /* entering capture range */
2639                         psDevInfo->abDumpedKCCBCtlAlready[eKCCBType] = IMG_TRUE;
2640
2641                         /* wait for firmware to catch up */
2642                         PVR_DPF((PVR_DBG_MESSAGE, "RGXSendCommandRaw: waiting on fw to catch-up. DM: %d, roff: %d, woff: %d",
2643                                                 eKCCBType, psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
2644                         PVRSRVPollForValueKM(&psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset, 0xFFFFFFFF);
2645
2646                         /* Dump Init state of Kernel CCB control (read and write offset) */
2647                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Initial state of kernel CCB Control(%d), roff: %d, woff: %d", eKCCBType, psKCCBCtl->ui32ReadOffset, psKCCBCtl->ui32WriteOffset);
2648                         DevmemPDumpLoadMem(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
2649                                         0,
2650                                         sizeof(RGXFWIF_CCB_CTL),
2651                                         PDUMP_FLAGS_CONTINUOUS);
2652                 }
2653         }
2654 #endif
2655
2656         PVR_ASSERT(ui32CmdSize == psKCCBCtl->ui32CmdSize);
2657         if (!OSLockIsLocked(PVRSRVGetPVRSRVData()->hPowerLock))
2658         {
2659                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandRaw called without power lock held!"));
2660                 PVR_ASSERT(OSLockIsLocked(PVRSRVGetPVRSRVData()->hPowerLock));
2661         }
2662
2663         /*
2664          * Acquire a slot in the CCB.
2665          */
2666         eError = RGXAcquireKernelCCBSlot(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType], psKCCBCtl, &ui32NewWriteOffset);
2667         if (eError != PVRSRV_OK)
2668         {
2669                 PVR_DPF((PVR_DBG_ERROR, "RGXSendCommandRaw failed to acquire CCB slot. Type:%u Error:%u",
2670                                 eKCCBType, eError));
2671                 goto _RGXSendCommandRaw_Exit;
2672         }
2673
2674         /*
2675          * Copy the command into the CCB.
2676          */
2677         OSMemCopy(&pui8KCCB[ui32OldWriteOffset * psKCCBCtl->ui32CmdSize],
2678                           psKCCBCmd, psKCCBCtl->ui32CmdSize);
2679
2680         /* ensure kCCB data is written before the offsets */
2681         OSWriteMemoryBarrier();
2682
2683         /* Move past the current command */
2684         psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
2685
2686
2687 #if defined(PDUMP)
2688         /* in capture range */
2689         if (bPdumpEnabled)
2690         {
2691                 /* Dump new Kernel CCB content */
2692                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Dump kCCB(%d) cmd, woff = %d", eKCCBType, ui32OldWriteOffset);
2693                 DevmemPDumpLoadMem(psDevInfo->apsKernelCCBMemDesc[eKCCBType],
2694                                 ui32OldWriteOffset * psKCCBCtl->ui32CmdSize,
2695                                 psKCCBCtl->ui32CmdSize,
2696                                 PDUMP_FLAGS_CONTINUOUS);
2697
2698                 /* Dump new kernel CCB write offset */
2699                 PDUMPCOMMENTWITHFLAGS(uiPdumpFlags, "Dump kCCBCtl(%d) woff: %d", eKCCBType, ui32NewWriteOffset);
2700                 DevmemPDumpLoadMem(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
2701                                                            offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
2702                                                            sizeof(IMG_UINT32),
2703                                                            uiPdumpFlags);
2704         }
2705
2706         /* out of capture range */
2707         if (!bPdumpEnabled)
2708         {
2709                 eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset, eKCCBType);
2710                 if (eError != PVRSRV_OK)
2711                 {
2712                         PVR_DPF((PVR_DBG_WARNING, "RGXSendCommandRaw: problem draining kCCB (%d)", eError));
2713                         goto _RGXSendCommandRaw_Exit;
2714                 }
2715         }
2716 #endif
2717
2718
2719         PDUMPCOMMENTWITHFLAGS(uiPdumpFlags, "MTS kick for kernel CCB %d", eKCCBType);
2720         /*
2721          * Kick the MTS to schedule the firmware.
2722          */
2723         {
2724                 IMG_UINT32      ui32MTSRegVal = (eKCCBType & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_COUNTED;
2725
2726                 __MTSScheduleWrite(psDevInfo, ui32MTSRegVal);
2727
2728                 PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE, ui32MTSRegVal, uiPdumpFlags);
2729         }
2730
2731 #if defined (NO_HARDWARE)
2732         /* keep the roff updated because fw isn't there to update it */
2733         psKCCBCtl->ui32ReadOffset = psKCCBCtl->ui32WriteOffset;
2734 #endif
2735
2736 _RGXSendCommandRaw_Exit:
2737         return eError;
2738 }
2739
2740 IMG_VOID RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
2741 {
2742         PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
2743         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
2744
2745         OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
2746 }
2747
2748 /*!
2749 ******************************************************************************
2750
2751  @Function      _RGXScheduleProcessQueuesMISR
2752
2753  @Description - Sends uncounted kick to all the DMs (the FW will process all
2754                                 the queue for all the DMs)
2755 ******************************************************************************/
2756 static IMG_VOID _RGXScheduleProcessQueuesMISR(IMG_VOID *pvData)
2757 {
2758         PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
2759         PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
2760         RGXFWIF_DM                         eDM;
2761         PVRSRV_ERROR               eError;
2762         PVRSRV_DEV_POWER_STATE ePowerState;
2763
2764         /* We don't need to acquire the BridgeLock as this power transition won't
2765            send a command to the FW */
2766         eError = PVRSRVPowerLock();
2767         if (eError != PVRSRV_OK)
2768         {
2769                 PVR_DPF((PVR_DBG_WARNING, "RGXScheduleProcessQueuesKM: failed to acquire powerlock (%s)",
2770                                         PVRSRVGetErrorStringKM(eError)));
2771
2772                 return;
2773         }
2774
2775         /* Check whether it's worth waking up the GPU */
2776         eError = PVRSRVGetDevicePowerState(psDeviceNode->sDevId.ui32DeviceIndex, &ePowerState);
2777
2778         if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
2779         {
2780                 RGXFWIF_GPU_UTIL_FWCB  *psUtilFWCb = psDevInfo->psRGXFWIfGpuUtilFWCb;
2781                 IMG_BOOL               bGPUHasWorkWaiting;
2782
2783                 bGPUHasWorkWaiting =
2784                     (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFWCb->ui64LastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
2785
2786                 if (!bGPUHasWorkWaiting)
2787                 {
2788                         /* all queues are empty, don't wake up the GPU */
2789                         PVRSRVPowerUnlock();
2790                         return;
2791                 }
2792         }
2793
2794         PDUMPPOWCMDSTART();
2795         /* wake up the GPU */
2796         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode->sDevId.ui32DeviceIndex,
2797                                                                                  PVRSRV_DEV_POWER_STATE_ON,
2798                                                                                  IMG_FALSE);
2799         PDUMPPOWCMDEND();
2800
2801         if (eError != PVRSRV_OK)
2802         {
2803                 PVR_DPF((PVR_DBG_WARNING, "RGXScheduleProcessQueuesKM: failed to transition Rogue to ON (%s)",
2804                                         PVRSRVGetErrorStringKM(eError)));
2805
2806                 PVRSRVPowerUnlock();
2807                 return;
2808         }
2809
2810         /* uncounted kick for all DMs */
2811         for (eDM = RGXFWIF_HWDM_MIN; eDM < RGXFWIF_HWDM_MAX; eDM++)
2812         {
2813                 IMG_UINT32      ui32MTSRegVal = (eDM & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
2814
2815                 __MTSScheduleWrite(psDevInfo, ui32MTSRegVal);
2816         }
2817
2818         PVRSRVPowerUnlock();
2819 }
2820
2821 PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
2822 {
2823         return OSInstallMISR(phMISR,
2824                              _RGXScheduleProcessQueuesMISR,
2825                              psDeviceNode);
2826 }
2827
2828 typedef struct _DEVMEM_COMMON_CONTEXT_LOOKUP_
2829 {
2830         IMG_UINT32                 ui32ContextID;
2831         RGX_SERVER_COMMON_CONTEXT  *psServerCommonContext;
2832 } DEVMEM_COMMON_CONTEXT_LOOKUP;
2833
2834
2835 static IMG_BOOL _FindServerCommonContext(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
2836 {
2837         DEVMEM_COMMON_CONTEXT_LOOKUP  *psRefLookUp = (DEVMEM_COMMON_CONTEXT_LOOKUP *)pvCallbackData;
2838         RGX_SERVER_COMMON_CONTEXT     *psServerCommonContext;
2839
2840         psServerCommonContext = IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
2841
2842         if (psServerCommonContext->ui32ContextID == psRefLookUp->ui32ContextID)
2843         {
2844                 psRefLookUp->psServerCommonContext = psServerCommonContext;
2845                 return IMG_FALSE;
2846         }
2847         else
2848         {
2849                 return IMG_TRUE;
2850         }
2851 }
2852
2853
2854 /*!
2855 ******************************************************************************
2856
2857  @Function      RGXScheduleCommand
2858
2859  @Description - Submits a CCB command and kicks the firmware but first schedules
2860                 any commands which have to happen before handle
2861
2862  @Input psDevInfo - pointer to device info
2863  @Input eKCCBType - see RGXFWIF_CMD_*
2864  @Input pvKCCBCmd - kernel CCB command
2865  @Input ui32CmdSize -
2866  @Input bPDumpContinuous - TRUE if the pdump flags should be continuous
2867
2868
2869  @Return ui32Error - success or failure
2870
2871 ******************************************************************************/
2872 PVRSRV_ERROR RGXScheduleCommand(PVRSRV_RGXDEV_INFO      *psDevInfo,
2873                                                                 RGXFWIF_DM                      eKCCBType,
2874                                                                 RGXFWIF_KCCB_CMD        *psKCCBCmd,
2875                                                                 IMG_UINT32                      ui32CmdSize,
2876                                                                 IMG_BOOL                        bPDumpContinuous)
2877 {
2878         PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
2879         PVRSRV_ERROR eError;
2880
2881         if ((eKCCBType == RGXFWIF_DM_3D) || (eKCCBType == RGXFWIF_DM_2D) || (eKCCBType == RGXFWIF_DM_CDM))
2882         {
2883                 /* This handles the no operation case */
2884                 OSCPUOperation(psData->uiCacheOp);
2885                 psData->uiCacheOp = PVRSRV_CACHE_OP_NONE;
2886         }
2887
2888         eError = RGXPreKickCacheCommand(psDevInfo);
2889         if (eError != PVRSRV_OK) goto RGXScheduleCommand_exit;
2890
2891         eError = RGXSendCommandWithPowLock(psDevInfo, eKCCBType, psKCCBCmd, ui32CmdSize, bPDumpContinuous);
2892         if (eError != PVRSRV_OK) goto RGXScheduleCommand_exit;
2893
2894
2895 RGXScheduleCommand_exit:
2896         return eError;
2897 }
2898
2899 /*
2900  * RGXCheckFirmwareCCBs
2901  */
2902 IMG_VOID RGXCheckFirmwareCCBs(PVRSRV_RGXDEV_INFO *psDevInfo)
2903 {
2904         RGXFWIF_FWCCB_CMD       *psFwCCBCmd;
2905         IMG_UINT32                      ui32DMCount;
2906
2907         for (ui32DMCount = 0; ui32DMCount < RGXFWIF_DM_MAX; ui32DMCount++)
2908         {
2909                 RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->apsFirmwareCCBCtl[ui32DMCount];
2910                 IMG_UINT8               *psFWCCB = psDevInfo->apsFirmwareCCB[ui32DMCount];
2911
2912                 while (psFWCCBCtl->ui32ReadOffset != psFWCCBCtl->ui32WriteOffset)
2913                 {
2914                         /* Point to the next command */
2915                         psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtl->ui32ReadOffset;
2916
2917                         switch(psFwCCBCmd->eCmdType)
2918                         {
2919                                 case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
2920                                 {
2921                                         if (psDevInfo->bPDPEnabled)
2922                                         {
2923                                                 PDUMP_PANIC(RGX, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
2924                                         }
2925                                         RGXProcessRequestZSBufferBacking(psDevInfo,
2926                                                                                                         psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
2927                                         break;
2928                                 }
2929
2930                                 case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
2931                                 {
2932                                         if (psDevInfo->bPDPEnabled)
2933                                         {
2934                                                 PDUMP_PANIC(RGX, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
2935                                         }
2936                                         RGXProcessRequestZSBufferUnbacking(psDevInfo,
2937                                                                                                         psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
2938                                         break;
2939                                 }
2940
2941                                 case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
2942                                 {
2943                                         if (psDevInfo->bPDPEnabled)
2944                                         {
2945                                                 PDUMP_PANIC(RGX, FREELIST_GROW, "Request to grow the free list");
2946                                         }
2947                                         RGXProcessRequestGrow(psDevInfo,
2948                                                                                 psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
2949                                         break;
2950                                 }
2951
2952                                 case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
2953                                 {
2954                                         if (psDevInfo->bPDPEnabled)
2955                                         {
2956                                                 PDUMP_PANIC(RGX, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
2957                                         }
2958
2959                                         PVR_DPF((PVR_DBG_MESSAGE, "RGXCheckFirmwareCCBs: Freelist reconstruction request (%d/%d) for %d freelists",
2960                                                 psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
2961                                                 psDevInfo->psRGXFWIfTraceBuf->ui32HwrCounter+1,
2962                                                 psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
2963
2964                                         RGXProcessRequestFreelistsReconstruction(psDevInfo, ui32DMCount,
2965                                                                                 psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
2966                                                                                 psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
2967                                         break;
2968                                 }
2969
2970                                 case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
2971                                 {
2972                                         DEVMEM_COMMON_CONTEXT_LOOKUP  sLookUp;
2973
2974                                         sLookUp.ui32ContextID         = psFwCCBCmd->uCmdData.sCmdContextResetNotification.ui32ServerCommonContextID;
2975                                         sLookUp.psServerCommonContext = IMG_NULL;
2976
2977                                         dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, _FindServerCommonContext, (IMG_PVOID)&sLookUp);
2978
2979                                         PVR_DPF((PVR_DBG_MESSAGE, "RGXCheckFirmwareCCBs: Context 0x%p reset (ID=0x%08x, Reason=%d)",
2980                                                 sLookUp.psServerCommonContext,
2981                                                 (IMG_UINT32)(psFwCCBCmd->uCmdData.sCmdContextResetNotification.ui32ServerCommonContextID),
2982                                                 (IMG_UINT32)(psFwCCBCmd->uCmdData.sCmdContextResetNotification.eResetReason)));
2983
2984                                         if (sLookUp.psServerCommonContext != IMG_NULL)
2985                                         {
2986                                                 sLookUp.psServerCommonContext->eLastResetReason = psFwCCBCmd->uCmdData.sCmdContextResetNotification.eResetReason;
2987                                         }
2988                                         break;
2989                                 }
2990
2991                                 case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
2992                                 {
2993                                         RGXDumpDebugInfo(IMG_NULL,psDevInfo);
2994                                         break;
2995                                 }
2996
2997                                 case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
2998                                 {
2999                     IMG_PID        pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
3000                     IMG_INT32  i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
3001
3002                     switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
3003                                         {
3004                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
3005                                                 {
3006                             PVRSRVStatsUpdateRenderContextStats(i32AdjustmentValue,0,0,0,0,0,pidTmp);
3007                                                         break;
3008                                                 }
3009                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
3010                                                 {
3011                                                         PVRSRVStatsUpdateRenderContextStats(0,i32AdjustmentValue,0,0,0,0,pidTmp);
3012                                                         break;
3013                                                 }
3014                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
3015                                                 {
3016                                                         PVRSRVStatsUpdateRenderContextStats(0,0,i32AdjustmentValue,0,0,0,pidTmp);
3017                                                         break;
3018                                                 }
3019                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
3020                                                 {
3021                                                         PVRSRVStatsUpdateRenderContextStats(0,0,0,i32AdjustmentValue,0,0,pidTmp);
3022                                                         break;
3023                                                 }
3024                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_SH_STORES:
3025                                                 {
3026                                                         PVRSRVStatsUpdateRenderContextStats(0,0,0,0,i32AdjustmentValue,0,pidTmp);
3027                                                         break;
3028                                                 }
3029                                                 case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
3030                                                 {
3031                                                         PVRSRVStatsUpdateRenderContextStats(0,0,0,0,0,i32AdjustmentValue,pidTmp);
3032                                                         break;
3033                                                 }
3034                                         }
3035                                         break;
3036                                 }
3037                                 default:
3038                                         PVR_ASSERT(IMG_FALSE);
3039                         }
3040
3041                         /* Update read offset */
3042                         psFWCCBCtl->ui32ReadOffset = (psFWCCBCtl->ui32ReadOffset + 1) & psFWCCBCtl->ui32WrapMask;
3043                 }
3044         }
3045 }
3046
3047 /*
3048  * PVRSRVRGXFrameworkCopyCommand
3049  */
3050 PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(DEVMEM_MEMDESC       *psFWFrameworkMemDesc,
3051                                                                                    IMG_PBYTE            pbyGPUFRegisterList,
3052                                                                                    IMG_UINT32           ui32FrameworkRegisterSize)
3053 {
3054         PVRSRV_ERROR    eError;
3055         RGXFWIF_RF_REGISTERS    *psRFReg;
3056
3057         eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
3058                                       (IMG_VOID **)&psRFReg);
3059         if (eError != PVRSRV_OK)
3060         {
3061                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXFrameworkCopyCommand: Failed to map firmware render context state (%u)",
3062                                 eError));
3063                 return eError;
3064         }
3065
3066         OSMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
3067
3068         /* Release the CPU mapping */
3069         DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
3070
3071         /*
3072          * Dump the FW framework buffer
3073          */
3074         PDUMPCOMMENT("Dump FWFramework buffer");
3075         DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
3076
3077         return PVRSRV_OK;
3078 }
3079
3080 /*
3081  * PVRSRVRGXFrameworkCreateKM
3082  */
3083 PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE      *psDeviceNode,
3084                                                                                 DEVMEM_MEMDESC          **ppsFWFrameworkMemDesc,
3085                                                                                 IMG_UINT32                      ui32FrameworkCommandSize)
3086 {
3087         PVRSRV_ERROR                    eError;
3088         PVRSRV_RGXDEV_INFO              *psDevInfo = psDeviceNode->pvDevice;
3089
3090         /*
3091                 Allocate device memory for the firmware GPU framework state.
3092                 Sufficient info to kick one or more DMs should be contained in this buffer
3093         */
3094         PDUMPCOMMENT("Allocate Rogue firmware framework state");
3095
3096         eError = DevmemFwAllocate(psDevInfo,
3097                                                           ui32FrameworkCommandSize,
3098                                                           RGX_FWCOMCTX_ALLOCFLAGS,
3099                                                           "FirmwareGPUFrameworkState",
3100                                                           ppsFWFrameworkMemDesc);
3101
3102         if (eError != PVRSRV_OK)
3103         {
3104                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXFrameworkContextKM: Failed to allocate firmware framework state (%u)",
3105                                 eError));
3106                 return eError;
3107         }
3108
3109         return PVRSRV_OK;
3110 }
3111
3112 PVRSRV_ERROR RGXWaitForFWOp(PVRSRV_RGXDEV_INFO  *psDevInfo,
3113                                                         RGXFWIF_DM eDM,
3114                                                         PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
3115                                                         IMG_BOOL bPDumpContinuous)
3116 {
3117         PVRSRV_ERROR            eError = PVRSRV_OK;
3118         PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
3119         RGXFWIF_KCCB_CMD        sCmdSyncPrim;
3120
3121         /* Ensure Rogue is powered up before kicking MTS */
3122         eError = PVRSRVPowerLock();
3123
3124         if (eError != PVRSRV_OK)
3125         {
3126                 PVR_DPF((PVR_DBG_ERROR, "RGXWaitForFWOp: failed to acquire powerlock (%s)",
3127                                         PVRSRVGetErrorStringKM(eError)));
3128
3129                 goto _PVRSRVPowerLock_Exit;
3130         }
3131
3132         PDUMPPOWCMDSTART();
3133         eError = PVRSRVSetDevicePowerStateKM(psDeviceNode->sDevId.ui32DeviceIndex,
3134                                                                                  PVRSRV_DEV_POWER_STATE_ON,
3135                                                                                  IMG_FALSE);
3136         PDUMPPOWCMDEND();
3137
3138         if (eError != PVRSRV_OK)
3139         {
3140                 PVR_DPF((PVR_DBG_ERROR, "RGXWaitForFWOp: failed to transition Rogue to ON (%s)",
3141                                         PVRSRVGetErrorStringKM(eError)));
3142
3143                 goto _PVRSRVSetDevicePowerStateKM_Exit;
3144         }
3145
3146
3147
3148
3149         /* Setup sync primitive */
3150         SyncPrimSet(psSyncPrim, 0);
3151
3152         /* prepare a sync command */
3153         sCmdSyncPrim.eCmdType = RGXFWIF_KCCB_CMD_SYNC;
3154         sCmdSyncPrim.uCmdData.sSyncData.uiSyncObjDevVAddr = SyncPrimGetFirmwareAddr(psSyncPrim);
3155         sCmdSyncPrim.uCmdData.sSyncData.uiUpdateVal = 1;
3156
3157         PDUMPCOMMENT("RGXWaitForFWOp: Submit Kernel SyncPrim [0x%08x] to DM %d ", sCmdSyncPrim.uCmdData.sSyncData.uiSyncObjDevVAddr, eDM);
3158
3159         /* submit the sync primitive to the kernel CCB */
3160         eError = RGXSendCommandRaw(psDevInfo,
3161                                                                 eDM,
3162                                                                 &sCmdSyncPrim,
3163                                                                 sizeof(RGXFWIF_KCCB_CMD),
3164                                                                 bPDumpContinuous  ? PDUMP_FLAGS_CONTINUOUS:0);
3165         if (eError != PVRSRV_OK)
3166         {
3167                 PVR_DPF((PVR_DBG_ERROR,"RGXScheduleCommandAndWait: Failed to schedule Kernel SyncPrim with error (%u)", eError));
3168                 goto _RGXSendCommandRaw_Exit;
3169         }
3170
3171         /* Wait for sync primitive to be updated */
3172 #if defined(PDUMP)
3173         PDUMPCOMMENT("RGXScheduleCommandAndWait: Poll for Kernel SyncPrim [0x%08x] on DM %d ", sCmdSyncPrim.uCmdData.sSyncData.uiSyncObjDevVAddr, eDM);
3174
3175         SyncPrimPDumpPol(psSyncPrim,
3176                                         1,
3177                                         0xffffffff,
3178                                         PDUMP_POLL_OPERATOR_EQUAL,
3179                                         bPDumpContinuous ? PDUMP_FLAGS_CONTINUOUS:0);
3180 #endif
3181
3182         {
3183                 RGXFWIF_CCB_CTL  *psKCCBCtl = psDevInfo->apsKernelCCBCtl[eDM];
3184                 IMG_UINT32       ui32CurrentQueueLength = (psKCCBCtl->ui32WrapMask+1 +
3185                                                                                                    psKCCBCtl->ui32WriteOffset -
3186                                                                                                    psKCCBCtl->ui32ReadOffset) & psKCCBCtl->ui32WrapMask;
3187                 IMG_UINT32       ui32MaxRetries;
3188
3189                 for (ui32MaxRetries = (ui32CurrentQueueLength + 1) * 3;
3190                          ui32MaxRetries > 0;
3191                          ui32MaxRetries--)
3192                 {
3193                         eError = PVRSRVWaitForValueKMAndHoldBridgeLockKM(psSyncPrim->pui32LinAddr, 1, 0xffffffff);
3194
3195                         if (eError != PVRSRV_ERROR_TIMEOUT)
3196                         {
3197                                 break;
3198                         }
3199                 }
3200
3201                 if (eError == PVRSRV_ERROR_TIMEOUT)
3202                 {
3203                         PVR_DPF((PVR_DBG_ERROR,"RGXScheduleCommandAndWait: PVRSRVWaitForValueKMAndHoldBridgeLock timed out. Dump debug information."));
3204                         PVRSRVPowerUnlock();
3205
3206                         PVRSRVDebugRequest(DEBUG_REQUEST_VERBOSITY_MAX,IMG_NULL);
3207                         PVR_ASSERT(eError != PVRSRV_ERROR_TIMEOUT);
3208                         goto _PVRSRVDebugRequest_Exit;
3209                 }
3210         }
3211
3212 _RGXSendCommandRaw_Exit:
3213 _PVRSRVSetDevicePowerStateKM_Exit:
3214
3215         PVRSRVPowerUnlock();
3216
3217 _PVRSRVDebugRequest_Exit:
3218 _PVRSRVPowerLock_Exit:
3219         return eError;
3220 }
3221
3222 static
3223 PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO       *psDevInfo,
3224                                                                            RGXFWIF_DM                   eDM,
3225                                                                            RGXFWIF_KCCB_CMD             *psKCCBCmd,
3226                                                                            IMG_UINT32                   ui32CmdSize,
3227                                                                            RGXFWIF_CLEANUP_TYPE eCleanupType,
3228                                                                            PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
3229                                                                            IMG_BOOL                             bPDumpContinuous)
3230 {
3231         PVRSRV_ERROR eError;
3232
3233         psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
3234
3235         psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
3236         psKCCBCmd->uCmdData.sCleanupData.uiSyncObjDevVAddr = SyncPrimGetFirmwareAddr(psSyncPrim);
3237
3238         SyncPrimSet(psSyncPrim, 0);
3239
3240         /*
3241                 Send the cleanup request to the firmware. If the resource is still busy
3242                 the firmware will tell us and we'll drop out with a retry.
3243         */
3244         eError = RGXScheduleCommand(psDevInfo,
3245                                                                 eDM,
3246                                                                 psKCCBCmd,
3247                                                                 ui32CmdSize,
3248                                                                 bPDumpContinuous);
3249         if (eError != PVRSRV_OK)
3250         {
3251                 goto fail_command;
3252         }
3253
3254         /* Wait for sync primitive to be updated */
3255 #if defined(PDUMP)
3256         PDUMPCOMMENT("Wait for the firmware to reply to the cleanup command");
3257         SyncPrimPDumpPol(psSyncPrim,
3258                                         RGXFWIF_CLEANUP_RUN,
3259                                         RGXFWIF_CLEANUP_RUN,
3260                                         PDUMP_POLL_OPERATOR_EQUAL,
3261                                         bPDumpContinuous ? PDUMP_FLAGS_CONTINUOUS:0);
3262
3263         /*
3264          * The cleanup request to the firmware will tell us if a given resource is busy or not.
3265          * If the RGXFWIF_CLEANUP_BUSY flag is set, this means that the resource is still in use.
3266          * In this case we return a PVRSRV_ERROR_RETRY error to the client drivers and they will
3267          * re-issue the cleanup request until it succeed.
3268          *
3269          * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
3270          * that cleanup requests are only submitted if the resource is unused.
3271          * If this is not the case, the following poll will block infinitely, making sure
3272          * the issue doesn't go unnoticed.
3273          */
3274         PDUMPCOMMENT("Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
3275                                         eDM,
3276                                         psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
3277                                         psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
3278         SyncPrimPDumpPol(psSyncPrim,
3279                                         0,
3280                                         RGXFWIF_CLEANUP_BUSY,
3281                                         PDUMP_POLL_OPERATOR_EQUAL,
3282                                         bPDumpContinuous ? PDUMP_FLAGS_CONTINUOUS:0);
3283 #endif
3284
3285         {
3286                 RGXFWIF_CCB_CTL  *psKCCBCtl = psDevInfo->apsKernelCCBCtl[eDM];
3287                 IMG_UINT32       ui32CurrentQueueLength = (psKCCBCtl->ui32WrapMask+1 +
3288                                                            psKCCBCtl->ui32WriteOffset -
3289                                                            psKCCBCtl->ui32ReadOffset) & psKCCBCtl->ui32WrapMask;
3290                 IMG_UINT32       ui32MaxRetries;
3291
3292                 for (ui32MaxRetries = ui32CurrentQueueLength + 1;
3293                          ui32MaxRetries > 0;
3294                          ui32MaxRetries--)
3295                 {
3296                         eError = PVRSRVWaitForValueKMAndHoldBridgeLockKM(psSyncPrim->pui32LinAddr, RGXFWIF_CLEANUP_RUN, RGXFWIF_CLEANUP_RUN);
3297
3298                         if (eError != PVRSRV_ERROR_TIMEOUT)
3299                         {
3300                                 break;
3301                         }
3302                 }
3303
3304                 /*
3305                         If the firmware hasn't got back to us in a timely manner
3306                         then bail and let the caller retry the command.
3307                 */
3308                 if (eError == PVRSRV_ERROR_TIMEOUT)
3309                 {
3310                         PVR_DPF((PVR_DBG_WARNING,"RGXScheduleCleanupCommand: PVRSRVWaitForValueKMAndHoldBridgeLock timed out. Dump debug information."));
3311
3312                         eError = PVRSRV_ERROR_RETRY;
3313 #if defined(DEBUG)
3314                             PVRSRVDebugRequest(DEBUG_REQUEST_VERBOSITY_MAX,IMG_NULL);
3315 #endif
3316                         goto fail_poll;
3317                 }
3318                 else if (eError != PVRSRV_OK)
3319                 {
3320                         goto fail_poll;
3321                 }
3322         }
3323
3324         /*
3325                 If the command has was run but a resource was busy, then the request
3326                 will need to be retried.
3327         */
3328         if (*psSyncPrim->pui32LinAddr & RGXFWIF_CLEANUP_BUSY)
3329         {
3330                 eError = PVRSRV_ERROR_RETRY;
3331                 goto fail_requestbusy;
3332         }
3333
3334         return PVRSRV_OK;
3335
3336 fail_requestbusy:
3337 fail_poll:
3338 fail_command:
3339         PVR_ASSERT(eError != PVRSRV_OK);
3340
3341         return eError;
3342 }
3343
3344 /*
3345         RGXRequestCommonContextCleanUp
3346 */
3347 PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
3348                                                                                           PRGXFWIF_FWCOMMONCONTEXT psFWCommonContextFWAddr,
3349                                                                                           PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim,
3350                                                                                           RGXFWIF_DM eDM)
3351 {
3352         RGXFWIF_KCCB_CMD                        sRCCleanUpCmd = {0};
3353         PVRSRV_ERROR                            eError;
3354
3355         PDUMPCOMMENT("Common ctx cleanup Request DM%d [context = 0x%08x]", eDM, psFWCommonContextFWAddr.ui32Addr);
3356
3357         /* Setup our command data, the cleanup call will fill in the rest */
3358         sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
3359
3360         /* Request cleanup of the firmware resource */
3361         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
3362                                                                            eDM,
3363                                                                            &sRCCleanUpCmd,
3364                                                                            sizeof(RGXFWIF_KCCB_CMD),
3365                                                                            RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
3366                                                                            psSyncPrim,
3367                                                                            IMG_FALSE);
3368
3369         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3370         {
3371                 PVR_DPF((PVR_DBG_ERROR,"RGXRequestCommonContextCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
3372         }
3373
3374         return eError;
3375 }
3376
3377 /*
3378  * RGXRequestHWRTDataCleanUp
3379  */
3380
3381 PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
3382                                                                                  PRGXFWIF_HWRTDATA psHWRTData,
3383                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync,
3384                                                                                  RGXFWIF_DM eDM)
3385 {
3386         RGXFWIF_KCCB_CMD                        sHWRTDataCleanUpCmd = {0};
3387         PVRSRV_ERROR                            eError;
3388
3389         PDUMPCOMMENT("HW RTData cleanup Request DM%d [HWRTData = 0x%08x]", eDM, psHWRTData.ui32Addr);
3390
3391         sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
3392
3393         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
3394                                                                            eDM,
3395                                                                            &sHWRTDataCleanUpCmd,
3396                                                                            sizeof(sHWRTDataCleanUpCmd),
3397                                                                            RGXFWIF_CLEANUP_HWRTDATA,
3398                                                                            psSync,
3399                                                                            IMG_FALSE);
3400
3401         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3402         {
3403                 PVR_DPF((PVR_DBG_ERROR,"RGXRequestHWRTDataCleanUp: Failed to schedule a HWRTData cleanup with error (%u)", eError));
3404         }
3405
3406         return eError;
3407 }
3408
3409 /*
3410         RGXFWRequestFreeListCleanUp
3411 */
3412 PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
3413                                                                                  PRGXFWIF_FREELIST psFWFreeList,
3414                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync)
3415 {
3416         RGXFWIF_KCCB_CMD                        sFLCleanUpCmd = {0};
3417         PVRSRV_ERROR                            eError;
3418
3419         PDUMPCOMMENT("Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
3420
3421         /* Setup our command data, the cleanup call will fill in the rest */
3422         sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
3423
3424         /* Request cleanup of the firmware resource */
3425         eError = RGXScheduleCleanupCommand(psDevInfo,
3426                                                                            RGXFWIF_DM_GP,
3427                                                                            &sFLCleanUpCmd,
3428                                                                            sizeof(RGXFWIF_KCCB_CMD),
3429                                                                            RGXFWIF_CLEANUP_FREELIST,
3430                                                                            psSync,
3431                                                                            IMG_FALSE);
3432
3433         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3434         {
3435                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestFreeListCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
3436         }
3437
3438         return eError;
3439 }
3440
3441 /*
3442         RGXFWRequestZSBufferCleanUp
3443 */
3444 PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
3445                                                                                  PRGXFWIF_ZSBUFFER psFWZSBuffer,
3446                                                                                  PVRSRV_CLIENT_SYNC_PRIM *psSync)
3447 {
3448         RGXFWIF_KCCB_CMD                        sZSBufferCleanUpCmd = {0};
3449         PVRSRV_ERROR                            eError;
3450
3451         PDUMPCOMMENT("ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
3452
3453         /* Setup our command data, the cleanup call will fill in the rest */
3454         sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
3455
3456         /* Request cleanup of the firmware resource */
3457         eError = RGXScheduleCleanupCommand(psDevInfo,
3458                                                                            RGXFWIF_DM_3D,
3459                                                                            &sZSBufferCleanUpCmd,
3460                                                                            sizeof(RGXFWIF_KCCB_CMD),
3461                                                                            RGXFWIF_CLEANUP_ZSBUFFER,
3462                                                                            psSync,
3463                                                                            IMG_FALSE);
3464
3465         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3466         {
3467                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestZSBufferCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
3468         }
3469
3470         return eError;
3471 }
3472
3473
3474 #if defined(RGX_FEATURE_RAY_TRACING)
3475 PVRSRV_ERROR RGXFWRequestRayFrameDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
3476                                                                                          PRGXFWIF_RAY_FRAME_DATA psHWFrameData,
3477                                                                                          PVRSRV_CLIENT_SYNC_PRIM *psSync,
3478                                                                                          RGXFWIF_DM eDM)
3479 {
3480         RGXFWIF_KCCB_CMD                        sHWFrameDataCleanUpCmd = {0};
3481         PVRSRV_ERROR                            eError;
3482
3483         PDUMPCOMMENT("HW FrameData cleanup Request DM%d [HWFrameData = 0x%08x]", eDM, psHWFrameData.ui32Addr);
3484
3485         sHWFrameDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWFrameData = psHWFrameData;
3486
3487         eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
3488                                                                            eDM,
3489                                                                            &sHWFrameDataCleanUpCmd,
3490                                                                            sizeof(sHWFrameDataCleanUpCmd),
3491                                                                            RGXFWIF_CLEANUP_HWFRAMEDATA,
3492                                                                            psSync,
3493                                                                            IMG_FALSE);
3494
3495         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3496         {
3497                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestRayFrameDataCleanUp: Failed to schedule a HWFrameData cleanup with error (%u)", eError));
3498         }
3499
3500         return eError;
3501 }
3502
3503 /*
3504         RGXFWRequestRPMFreeListCleanUp
3505 */
3506 PVRSRV_ERROR RGXFWRequestRPMFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
3507                                                                                         PRGXFWIF_RPM_FREELIST psFWRPMFreeList,
3508                                                                                         PVRSRV_CLIENT_SYNC_PRIM *psSync)
3509 {
3510         RGXFWIF_KCCB_CMD                        sFLCleanUpCmd = {0};
3511         PVRSRV_ERROR                            eError;
3512
3513         PDUMPCOMMENT("RPM Free list cleanup Request [RPM FreeList = 0x%08x]", psFWRPMFreeList.ui32Addr);
3514
3515         /* Setup our command data, the cleanup call will fill in the rest */
3516         sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psRPMFreelist = psFWRPMFreeList;
3517
3518         /* Request cleanup of the firmware resource */
3519         eError = RGXScheduleCleanupCommand(psDevInfo,
3520                                                                            RGXFWIF_DM_GP,
3521                                                                            &sFLCleanUpCmd,
3522                                                                            sizeof(RGXFWIF_KCCB_CMD),
3523                                                                            RGXFWIF_CLEANUP_RPM_FREELIST,
3524                                                                            psSync,
3525                                                                            IMG_FALSE);
3526
3527         if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
3528         {
3529                 PVR_DPF((PVR_DBG_ERROR,"RGXFWRequestRPMFreeListCleanUp: Failed to schedule a memory context cleanup with error (%u)", eError));
3530         }
3531
3532         return eError;
3533 }
3534 #endif
3535
3536 PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
3537                                                                 CONNECTION_DATA *psConnection,
3538                                                                 PVRSRV_RGXDEV_INFO *psDevInfo,
3539                                                                 IMG_UINT32 ui32Priority,
3540                                                                 RGXFWIF_DM eDM)
3541 {
3542         IMG_UINT32                              ui32CmdSize;
3543         IMG_UINT8                               *pui8CmdPtr;
3544         RGXFWIF_KCCB_CMD                sPriorityCmd;
3545         RGXFWIF_CCB_CMD_HEADER  *psCmdHeader;
3546         RGXFWIF_CMD_PRIORITY    *psCmd;
3547         IMG_UINT32                              ui32BeforeWOff = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psContext));
3548         IMG_BOOL                                bKickCMD = IMG_TRUE;
3549         PVRSRV_ERROR                    eError;
3550
3551         /*
3552                 Get space for command
3553         */
3554         ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_CMD_PRIORITY));
3555
3556         eError = RGXAcquireCCB(FWCommonContextGetClientCCB(psContext),
3557                                                    ui32CmdSize,
3558                                                    (IMG_PVOID *) &pui8CmdPtr,
3559                                                    IMG_TRUE);
3560         if (eError != PVRSRV_OK)
3561         {
3562                 if (ui32BeforeWOff != RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psContext)))
3563                 {
3564                         bKickCMD = IMG_FALSE;
3565                 }
3566                 else
3567                 {
3568                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire client CCB", __FUNCTION__));
3569                         goto fail_ccbacquire;
3570                 }
3571         }
3572
3573         if (bKickCMD)
3574         {
3575                 /*
3576                         Write the command header and command
3577                 */
3578                 psCmdHeader = (RGXFWIF_CCB_CMD_HEADER *) pui8CmdPtr;
3579                 psCmdHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PRIORITY;
3580                 psCmdHeader->ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CMD_PRIORITY));
3581                 pui8CmdPtr += sizeof(*psCmdHeader);
3582
3583                 psCmd = (RGXFWIF_CMD_PRIORITY *) pui8CmdPtr;
3584                 psCmd->ui32Priority = ui32Priority;
3585                 pui8CmdPtr += sizeof(*psCmd);
3586         }
3587
3588         /*
3589                 We should reserved space in the kernel CCB here and fill in the command
3590                 directly.
3591                 This is so if there isn't space in the kernel CCB we can return with
3592                 retry back to services client before we take any operations
3593         */
3594
3595         if (bKickCMD)
3596         {
3597                 /*
3598                         Submit the command
3599                 */
3600                 RGXReleaseCCB(FWCommonContextGetClientCCB(psContext),
3601                                           ui32CmdSize,
3602                                           IMG_TRUE);
3603
3604                 if (eError != PVRSRV_OK)
3605                 {
3606                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release space in client CCB", __FUNCTION__));
3607                         return eError;
3608                 }
3609         }
3610
3611         /* Construct the priority command. */
3612         sPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
3613         sPriorityCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psContext);
3614         sPriorityCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psContext));
3615         sPriorityCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
3616
3617         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3618         {
3619                 eError = RGXScheduleCommand(psDevInfo,
3620                                                                         eDM,
3621                                                                         &sPriorityCmd,
3622                                                                         sizeof(sPriorityCmd),
3623                                                                         IMG_TRUE);
3624                 if (eError != PVRSRV_ERROR_RETRY)
3625                 {
3626                         break;
3627                 }
3628                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3629         } END_LOOP_UNTIL_TIMEOUT();
3630
3631         if (eError != PVRSRV_OK)
3632         {
3633                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXFlushComputeDataKM: Failed to schedule SLC flush command with error (%u)", eError));
3634         }
3635
3636         return PVRSRV_OK;
3637
3638 fail_ccbacquire:
3639         PVR_ASSERT(eError != PVRSRV_OK);
3640         return eError;
3641 }
3642
3643 /*
3644         RGXReadMETAAddr
3645 */
3646 PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
3647 {
3648         IMG_UINT8 *pui8RegBase = (IMG_UINT8*)psDevInfo->pvRegsBaseKM;
3649         IMG_UINT32 ui32Value;
3650
3651         /* Wait for Slave Port to be Ready */
3652         if (PVRSRVPollForValueKM(
3653                 (IMG_UINT32*) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
3654                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
3655                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN) != PVRSRV_OK)
3656         {
3657                 return PVRSRV_ERROR_TIMEOUT;
3658         }
3659
3660         /* Issue the Read */
3661         OSWriteHWReg32(
3662             psDevInfo->pvRegsBaseKM,
3663             RGX_CR_META_SP_MSLVCTRL0,
3664             ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
3665
3666         /* Wait for Slave Port to be Ready: read complete */
3667         if (PVRSRVPollForValueKM(
3668                 (IMG_UINT32*) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
3669                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
3670                 RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN) != PVRSRV_OK)
3671         {
3672                 return PVRSRV_ERROR_TIMEOUT;
3673         }
3674
3675         /* Read the value */
3676         ui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
3677
3678         *pui32Value = ui32Value;
3679
3680         return PVRSRV_OK;
3681 }
3682
3683
3684 /*
3685         RGXUpdateHealthStatus
3686 */
3687 PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
3688                                    IMG_BOOL bCheckAfterTimePassed)
3689 {
3690         PVRSRV_DATA*                 psPVRSRVData = PVRSRVGetPVRSRVData();
3691         PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
3692         PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
3693         PVRSRV_RGXDEV_INFO*  psDevInfo;
3694         RGXFWIF_TRACEBUF*  psRGXFWIfTraceBufCtl;
3695         IMG_UINT32  ui32DMCount, ui32ThreadCount;
3696         IMG_BOOL  bKCCBCmdsWaiting;
3697
3698         PVR_ASSERT(psDevNode != NULL);
3699         psDevInfo = psDevNode->pvDevice;
3700         psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
3701
3702         /* If the firmware is not initialised, there is not much point continuing! */
3703         if (!psDevInfo->bFirmwareInitialised  ||  psDevInfo->pvRegsBaseKM == IMG_NULL  ||
3704             psDevInfo->psDeviceNode == IMG_NULL)
3705         {
3706                 return PVRSRV_OK;
3707         }
3708
3709         /* If Rogue is not powered on, don't continue
3710            (there is a race condition where PVRSRVIsDevicePowered returns TRUE when the GPU is actually powering down.
3711            That's not a problem as this function does not touch the HW except for the RGXScheduleCommand function,
3712            which is already powerlock safe. The worst thing that could happen is that Rogue might power back up
3713            but the chances of that are very low */
3714         if (!PVRSRVIsDevicePowered(psDevNode->sDevId.ui32DeviceIndex))
3715         {
3716                 return PVRSRV_OK;
3717         }
3718
3719         /* If this is a quick update, then include the last current value... */
3720         if (!bCheckAfterTimePassed)
3721         {
3722                 eNewStatus = psDevNode->eHealthStatus;
3723                 eNewReason = psDevNode->eHealthReason;
3724         }
3725
3726         /*
3727            Firmware thread checks...
3728         */
3729         for (ui32ThreadCount = 0;  ui32ThreadCount < RGXFW_THREAD_NUM;  ui32ThreadCount++)
3730         {
3731                 if (psRGXFWIfTraceBufCtl != IMG_NULL)
3732                 {
3733                         IMG_CHAR*  pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
3734
3735                         /*
3736                         Check if the FW has hit an assert...
3737                         */
3738                         if (*pszTraceAssertInfo != '\0')
3739                         {
3740                                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Firmware thread %d has asserted: %s (%s:%d)",
3741                                         ui32ThreadCount, pszTraceAssertInfo,
3742                                                 psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
3743                                                 psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
3744                                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
3745                                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
3746                                 goto _RGXUpdateHealthStatus_Exit;
3747                         }
3748
3749                         /*
3750                            Check the threads to see if they are in the same poll locations as last time...
3751                         */
3752                         if (bCheckAfterTimePassed)
3753                         {
3754                                 if (psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] != 0  &&
3755                                         psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] == psDevInfo->aui32CrLastPollAddr[ui32ThreadCount])
3756                                 {
3757                                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
3758                                                         ui32ThreadCount,
3759                                                         ((psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
3760                                                         psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
3761                                                         psRGXFWIfTraceBufCtl->aui32CrPollMask[ui32ThreadCount]));
3762                                         eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
3763                                         eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
3764                                         goto _RGXUpdateHealthStatus_Exit;
3765                                 }
3766                                 psDevInfo->aui32CrLastPollAddr[ui32ThreadCount] = psRGXFWIfTraceBufCtl->aui32CrPollAddr[ui32ThreadCount];
3767                         }
3768                 }
3769         }
3770
3771         /*
3772            Event Object Timeouts check...
3773         */
3774         if (psDevInfo->ui32GEOTimeoutsLastTime > 1  &&  psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
3775         {
3776                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Global Event Object Timeouts have risen (from %d to %d)",
3777                                 psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
3778                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
3779                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
3780         }
3781         psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
3782
3783         /*
3784            Check the Kernel CCB pointers are valid. If any commands were waiting last time, then check
3785            that some have executed since then.
3786         */
3787         bKCCBCmdsWaiting = IMG_FALSE;
3788
3789         for (ui32DMCount = 0; ui32DMCount < RGXFWIF_DM_MAX; ui32DMCount++)
3790         {
3791                 RGXFWIF_CCB_CTL *psKCCBCtl = ((PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice)->apsKernelCCBCtl[ui32DMCount];
3792
3793                 if (psKCCBCtl != IMG_NULL)
3794                 {
3795                         if (psKCCBCtl->ui32ReadOffset > psKCCBCtl->ui32WrapMask  ||
3796                                 psKCCBCtl->ui32WriteOffset > psKCCBCtl->ui32WrapMask)
3797                         {
3798                                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: KCCB for DM%d has invalid offset (ROFF=%d WOFF=%d)",
3799                                         ui32DMCount, psKCCBCtl->ui32ReadOffset, psKCCBCtl->ui32WriteOffset));
3800                                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
3801                                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
3802                         }
3803
3804                         if (psKCCBCtl->ui32ReadOffset != psKCCBCtl->ui32WriteOffset)
3805                         {
3806                                 bKCCBCmdsWaiting = IMG_TRUE;
3807                         }
3808                 }
3809         }
3810
3811         if (bCheckAfterTimePassed && psDevInfo->psRGXFWIfTraceBuf != IMG_NULL)
3812         {
3813                 IMG_UINT32  ui32KCCBCmdsExecuted = psDevInfo->psRGXFWIfTraceBuf->ui32KCCBCmdsExecuted;
3814
3815                 if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
3816                 {
3817                         /*
3818                            If something was waiting last time then the Firmware has stopped processing commands.
3819                         */
3820                         if (psDevInfo->bKCCBCmdsWaitingLastTime)
3821                         {
3822                                 PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: No KCCB commands executed since check!"));
3823                                 eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
3824                                 eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
3825                         }
3826
3827                         /*
3828                            If no commands are currently pending and nothing happened since the last poll, then
3829                            schedule a dummy command to ping the firmware so we know it is alive and processing.
3830                         */
3831                         if (!bKCCBCmdsWaiting)
3832                         {
3833                                 RGXFWIF_KCCB_CMD  sCmpKCCBCmd;
3834                                 PVRSRV_ERROR      eError;
3835
3836                                 sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
3837
3838                                 eError = RGXScheduleCommand(psDevNode->pvDevice,
3839                                                                                         RGXFWIF_DM_GP,
3840                                                                                         &sCmpKCCBCmd,
3841                                                                                         sizeof(sCmpKCCBCmd),
3842                                                                                         IMG_TRUE);
3843                                 if (eError != PVRSRV_OK)
3844                                 {
3845                                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Cannot schedule Health Check command! (0x%x)", eError));
3846                                 }
3847                                 else
3848                                 {
3849                                         bKCCBCmdsWaiting = IMG_TRUE;
3850                                 }
3851                         }
3852                 }
3853
3854                 psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
3855                 psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
3856         }
3857
3858         if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
3859         {
3860                 /* Attempt to detect and deal with any stalled client contexts */
3861                 IMG_BOOL bStalledClient = IMG_FALSE;
3862                 if (CheckForStalledClientTransferCtxt(psDevInfo))
3863                 {
3864                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Detected stalled client transfer context"));
3865                         bStalledClient = IMG_TRUE;
3866                 }
3867                 if (CheckForStalledClientRenderCtxt(psDevInfo))
3868                 {
3869                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Detected stalled client render context"));
3870                         bStalledClient = IMG_TRUE;
3871                 }
3872 #if !defined(UNDER_WDDM)
3873                 if (CheckForStalledClientComputeCtxt(psDevInfo))
3874                 {
3875                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Detected stalled client compute context"));
3876                         bStalledClient = IMG_TRUE;
3877                 }
3878 #endif
3879 #if defined(RGX_FEATURE_RAY_TRACING)
3880                 if (CheckForStalledClientRayCtxt(psDevInfo))
3881                 {
3882                         PVR_DPF((PVR_DBG_WARNING, "RGXGetDeviceHealthStatus: Detected stalled client raytrace context"));
3883                         bStalledClient = IMG_TRUE;
3884                 }
3885 #endif
3886                 /* try the unblock routines only on the transition from OK to stalled */
3887                 if (!psDevInfo->bStalledClient && bStalledClient)
3888                 {
3889 #if defined(SUPPORT_DISPLAY_CLASS)
3890                         //DCDisplayContextFlush();
3891 #endif
3892                 }
3893                 psDevInfo->bStalledClient = bStalledClient;
3894         }
3895
3896         /*
3897            Finished, save the new status...
3898         */
3899 _RGXUpdateHealthStatus_Exit:
3900         psDevNode->eHealthStatus = eNewStatus;
3901         psDevNode->eHealthReason = eNewReason;
3902
3903         return PVRSRV_OK;
3904 } /* RGXUpdateHealthStatus */
3905
3906 PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext)
3907 {
3908         RGX_CLIENT_CCB  *psCurrentClientCCB = psCurrentServerCommonContext->psClientCCB;
3909
3910         return CheckForStalledCCB(psCurrentClientCCB);
3911 }
3912
3913 IMG_VOID DumpStalledFWCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
3914                                                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf)
3915 {
3916         RGX_CLIENT_CCB  *psCurrentClientCCB = psCurrentServerCommonContext->psClientCCB;
3917         PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext = psCurrentServerCommonContext->sFWCommonContextFWAddr;
3918
3919 #if defined(PVRSRV_ENABLE_FULL_SYNC_TRACKING) || defined(PVRSRV_ENABLE_FULL_CCB_DUMP)
3920         DumpCCB(sFWCommonContext, psCurrentClientCCB, pfnDumpDebugPrintf);
3921 #else
3922         DumpStalledCCBCommand(sFWCommonContext, psCurrentClientCCB, pfnDumpDebugPrintf);
3923 #endif
3924 }
3925
3926 IMG_VOID AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
3927                                                                         IMG_UINT32 *pui32NumCleanupCtl,
3928                                                                         RGXFWIF_DM eDM,
3929                                                                         IMG_BOOL bKick,
3930                                                                         RGX_RTDATA_CLEANUP_DATA        *psRTDataCleanup,
3931                                                                         RGX_ZSBUFFER_DATA              *psZBuffer,
3932                                                                         RGX_ZSBUFFER_DATA              *psSBuffer)
3933 {
3934         PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
3935
3936         PVR_ASSERT((eDM == RGXFWIF_DM_TA) || (eDM == RGXFWIF_DM_3D));
3937
3938         if(bKick)
3939         {
3940                 if(eDM == RGXFWIF_DM_TA)
3941                 {
3942                         if(psRTDataCleanup)
3943                         {
3944                                 PRGXFWIF_CLEANUP_CTL psCleanupCtl;
3945
3946                                 RGXSetFirmwareAddress(&psCleanupCtl, psRTDataCleanup->psFWHWRTDataMemDesc,
3947                                                                         offsetof(RGXFWIF_HWRTDATA, sTACleanupState),
3948                                                                 RFW_FWADDR_NOREF_FLAG);
3949
3950                                 *(psCleanupCtlWrite++) = psCleanupCtl;
3951                         }
3952                 }
3953                 else
3954                 {
3955                         if(psRTDataCleanup)
3956                         {
3957                                 PRGXFWIF_CLEANUP_CTL psCleanupCtl;
3958
3959                                 RGXSetFirmwareAddress(&psCleanupCtl, psRTDataCleanup->psFWHWRTDataMemDesc,
3960                                                                         offsetof(RGXFWIF_HWRTDATA, s3DCleanupState),
3961                                                                 RFW_FWADDR_NOREF_FLAG);
3962
3963                                 *(psCleanupCtlWrite++) = psCleanupCtl;
3964                         }
3965
3966                         if(psZBuffer)
3967                         {
3968                                 (psCleanupCtlWrite++)->ui32Addr = psZBuffer->sZSBufferFWDevVAddr.ui32Addr +
3969                                                                 offsetof(RGXFWIF_FWZSBUFFER, sCleanupState);
3970                         }
3971
3972                         if(psSBuffer)
3973                         {
3974                                 (psCleanupCtlWrite++)->ui32Addr = psSBuffer->sZSBufferFWDevVAddr.ui32Addr +
3975                                                                 offsetof(RGXFWIF_FWZSBUFFER, sCleanupState);
3976                         }
3977                 }
3978         }
3979
3980         *pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
3981
3982         PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
3983 }
3984
3985 PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
3986 {
3987         PVRSRV_RGXDEV_INFO      *psDevInfo;
3988         RGXFWIF_HWRINFOBUF      *psHWRInfoBuf;
3989         RGXFWIF_TRACEBUF        *psRGXFWIfTraceBufCtl;
3990         IMG_UINT32                      i;
3991
3992         if(psDevNode->pvDevice == IMG_NULL)
3993         {
3994                 return PVRSRV_ERROR_INVALID_DEVINFO;
3995         }
3996         psDevInfo = psDevNode->pvDevice;
3997
3998         psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBuf;
3999         psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBuf;
4000
4001         for(i = 0 ; i < RGXFWIF_DM_MAX ; i++)
4002         {
4003                 /* Reset the HWR numbers */
4004                 psRGXFWIfTraceBufCtl->aui16HwrDmLockedUpCount[i] = 0;
4005                 psRGXFWIfTraceBufCtl->aui16HwrDmFalseDetectCount[i] = 0;
4006                 psRGXFWIfTraceBufCtl->aui16HwrDmRecoveredCount[i] = 0;
4007                 psRGXFWIfTraceBufCtl->aui16HwrDmOverranCount[i] = 0;
4008         }
4009
4010         for(i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
4011         {
4012                 psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
4013         }
4014
4015         for(i = 0 ; i < RGXFW_THREAD_NUM ; i++)
4016         {
4017                 psHWRInfoBuf->ui32FirstCrPollAddr[i] = 0;
4018                 psHWRInfoBuf->ui32FirstCrPollMask[i] = 0;
4019         }
4020
4021         psHWRInfoBuf->ui32WriteIndex = 0;
4022         psHWRInfoBuf->ui32DDReqCount = 0;
4023
4024         return PVRSRV_OK;
4025 }
4026
4027 #if defined(PDUMP)
4028 PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset, RGXFWIF_DM eKCCBType)
4029 {
4030         RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->apsKernelCCBCtl[eKCCBType];
4031         PVRSRV_ERROR eError = PVRSRV_OK;
4032
4033         if (psDevInfo->abDumpedKCCBCtlAlready[eKCCBType])
4034         {
4035                 /* exiting capture range */
4036                 psDevInfo->abDumpedKCCBCtlAlready[eKCCBType] = IMG_FALSE;
4037
4038                 /* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
4039                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
4040                                       "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
4041                                       psKCCBCtl,
4042                                       ui32WriteOffset,
4043                                       ui32WriteOffset);
4044                 eError = DevmemPDumpDevmemPol32(psDevInfo->apsKernelCCBCtlMemDesc[eKCCBType],
4045                                                 offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
4046                                                 ui32WriteOffset,
4047                                                 0xffffffff,
4048                                                 PDUMP_POLL_OPERATOR_EQUAL,
4049                                                 PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
4050
4051                 if (eError != PVRSRV_OK)
4052                 {
4053                         PVR_DPF((PVR_DBG_ERROR, "RGXPdumpDrainKCCB: problem pdumping POL for kCCBCtl (%d)", eError));
4054                 }
4055         }
4056         return eError;
4057 }
4058 #endif
4059
4060 /******************************************************************************
4061  End of file (rgxfwutils.c)
4062 ******************************************************************************/