drivers/gpu/rogue/services/server/devices/rgx/rgxta3d.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          RGX TA/3D routines
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    RGX TA/3D routines
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43 /* for the offsetof macro */
  44 #include <stddef.h>
  45
  46 #include "pdump_km.h"
  47 #include "pvr_debug.h"
  48 #include "rgxutils.h"
  49 #include "rgxfwutils.h"
  50 #include "rgxta3d.h"
  51 #include "rgxmem.h"
  52 #include "allocmem.h"
  53 #include "devicemem.h"
  54 #include "devicemem_pdump.h"
  55 #include "osfunc.h"
  56 #include "pvrsrv.h"
  57 #include "rgx_memallocflags.h"
  58 #include "rgxccb.h"
  59 #include "rgxhwperf.h"
  60 #include "rgxtimerquery.h"
  61 #include "rgxsync.h"
  62
  63 #include "rgxdefs_km.h"
  64 #include "rgx_fwif_km.h"
  65 #include "physmem.h"
  66 #include "sync_server.h"
  67 #include "sync_internal.h"
  68 #include "process_stats.h"
  69 #include "osfunc.h"
  70
  71 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
  72 #include "pvr_sync.h"
  73 #endif /* defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) */
  74
  75 typedef struct _DEVMEM_REF_LOOKUP_
  76 {
  77         IMG_UINT32 ui32ZSBufferID;
  78         RGX_ZSBUFFER_DATA *psZSBuffer;
  79 } DEVMEM_REF_LOOKUP;
  80
  81 typedef struct _DEVMEM_FREELIST_LOOKUP_
  82 {
  83         IMG_UINT32 ui32FreeListID;
  84         RGX_FREELIST *psFreeList;
  85 } DEVMEM_FREELIST_LOOKUP;
  86
  87 typedef struct {
  88         DEVMEM_MEMDESC                          *psContextStateMemDesc;
  89         RGX_SERVER_COMMON_CONTEXT       *psServerCommonContext;
  90         IMG_UINT32                                      ui32Priority;
  91 } RGX_SERVER_RC_TA_DATA;
  92
  93 typedef struct {
  94         DEVMEM_MEMDESC                          *psContextStateMemDesc;
  95         RGX_SERVER_COMMON_CONTEXT       *psServerCommonContext;
  96         IMG_UINT32                                      ui32Priority;
  97 } RGX_SERVER_RC_3D_DATA;
  98
  99 struct _RGX_SERVER_RENDER_CONTEXT_ {
 100         PVRSRV_DEVICE_NODE                      *psDeviceNode;
 101         DEVMEM_MEMDESC                          *psFWRenderContextMemDesc;
 102         DEVMEM_MEMDESC                          *psFWFrameworkMemDesc;
 103         RGX_SERVER_RC_TA_DATA           sTAData;
 104         RGX_SERVER_RC_3D_DATA           s3DData;
 105         IMG_UINT32                                      ui32CleanupStatus;
 106 #define RC_CLEANUP_TA_COMPLETE          (1 << 0)
 107 #define RC_CLEANUP_3D_COMPLETE          (1 << 1)
 108         PVRSRV_CLIENT_SYNC_PRIM         *psCleanupSync;
 109         DLLIST_NODE                                     sListNode;
 110         SYNC_ADDR_LIST                  sSyncAddrListTAFence;
 111         SYNC_ADDR_LIST                  sSyncAddrListTAUpdate;
 112         SYNC_ADDR_LIST                  sSyncAddrList3DFence;
 113         SYNC_ADDR_LIST                  sSyncAddrList3DUpdate;
 114 };
 115
 116
 117 static
 118 #ifdef __GNUC__
 119         __attribute__((noreturn))
 120 #endif
 121 void sleep_for_ever(void)
 122 {
 123 #if defined(__KLOCWORK__) // klocworks would report an infinite loop because of while(1).
 124         PVR_ASSERT(0);
 125 #else
 126         while(1)
 127         {
 128                 OSSleepms(~0); // sleep the maximum amount of time possible
 129         }
 130 #endif
 131 }
 132
 133
 134 /*
 135         Static functions used by render context code
 136 */
 137
 138 static
 139 PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
 140                                                            PVRSRV_DEVICE_NODE *psDeviceNode,
 141                                                            PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
 142 {
 143         PVRSRV_ERROR eError;
 144
 145         /* Check if the FW has finished with this resource ... */
 146         eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
 147                                                                                           FWCommonContextGetFWAddress(psTAData->psServerCommonContext),
 148                                                                                           psCleanupSync,
 149                                                                                           RGXFWIF_DM_TA);
 150         if (eError == PVRSRV_ERROR_RETRY)
 151         {
 152                 return eError;
 153         }
 154         else if (eError != PVRSRV_OK)
 155         {
 156                 PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
 157                                 __FUNCTION__,
 158                                 PVRSRVGetErrorStringKM(eError)));
 159                 return eError;
 160         }
 161
 162         /* ... it has so we can free it's resources */
 163 #if defined(DEBUG)
 164         /* Log the number of TA context stores which occurred */
 165         {
 166                 RGXFWIF_TACTX_STATE     *psFWTAState;
 167
 168                 eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
 169                                                                                   (IMG_VOID**)&psFWTAState);
 170                 if (eError != PVRSRV_OK)
 171                 {
 172                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware render context state (%u)",
 173                                         __FUNCTION__, eError));
 174                 }
 175                 else
 176                 {
 177                         /* Release the CPU virt addr */
 178                         DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
 179                 }
 180         }
 181 #endif
 182         FWCommonContextFree(psTAData->psServerCommonContext);
 183         DevmemFwFree(psTAData->psContextStateMemDesc);
 184         return PVRSRV_OK;
 185 }
 186
 187 static
 188 PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
 189                                                            PVRSRV_DEVICE_NODE *psDeviceNode,
 190                                                            PVRSRV_CLIENT_SYNC_PRIM *psCleanupSync)
 191 {
 192         PVRSRV_ERROR eError;
 193
 194         /* Check if the FW has finished with this resource ... */
 195         eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
 196                                                                                           FWCommonContextGetFWAddress(ps3DData->psServerCommonContext),
 197                                                                                           psCleanupSync,
 198                                                                                           RGXFWIF_DM_3D);
 199         if (eError == PVRSRV_ERROR_RETRY)
 200         {
 201                 return eError;
 202         }
 203         else if (eError != PVRSRV_OK)
 204         {
 205                 PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
 206                                  __FUNCTION__,
 207                                  PVRSRVGetErrorStringKM(eError)));
 208                 return eError;
 209         }
 210
 211         /* ... it has so we can free it's resources */
 212 #if defined(DEBUG)
 213         /* Log the number of 3D context stores which occurred */
 214         {
 215                 RGXFWIF_3DCTX_STATE     *psFW3DState;
 216
 217                 eError = DevmemAcquireCpuVirtAddr(ps3DData->psContextStateMemDesc,
 218                                                                                   (IMG_VOID**)&psFW3DState);
 219                 if (eError != PVRSRV_OK)
 220                 {
 221                         PVR_DPF((PVR_DBG_ERROR,"%s: Failed to map firmware render context state (%u)",
 222                                         __FUNCTION__, eError));
 223                 }
 224                 else
 225                 {
 226                         /* Release the CPU virt addr */
 227                         DevmemReleaseCpuVirtAddr(ps3DData->psContextStateMemDesc);
 228                 }
 229         }
 230 #endif
 231
 232         FWCommonContextFree(ps3DData->psServerCommonContext);
 233         DevmemFwFree(ps3DData->psContextStateMemDesc);
 234         return PVRSRV_OK;
 235 }
 236
 237 static IMG_BOOL _RGXDumpPMRPageList(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
 238 {
 239         RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 240         PVRSRV_ERROR                    eError;
 241
 242         eError = PMRDumpPageList(psPMRNode->psPMR,
 243                                                         RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 244         if (eError != PVRSRV_OK)
 245         {
 246                 PVR_DPF((PVR_DBG_ERROR,"Error (%u) printing pmr %p", eError, psPMRNode->psPMR));
 247         }
 248
 249         return IMG_TRUE;
 250 }
 251
 252 IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
 253 {
 254         PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016llx",
 255                                 psFreeList->sFreeListFWDevVAddr.ui32Addr,
 256                                 psFreeList->ui32FreelistID,
 257                                 psFreeList->ui64FreelistChecksum));
 258
 259         /* Dump Init FreeList page list */
 260         PVR_LOG(("  Initial Memory block"));
 261         dllist_foreach_node(&psFreeList->sMemoryBlockInitHead,
 262                                         _RGXDumpPMRPageList,
 263                                         IMG_NULL);
 264
 265         /* Dump Grow FreeList page list */
 266         PVR_LOG(("  Grow Memory blocks"));
 267         dllist_foreach_node(&psFreeList->sMemoryBlockHead,
 268                                         _RGXDumpPMRPageList,
 269                                         IMG_NULL);
 270
 271         return IMG_TRUE;
 272 }
 273
 274 static PVRSRV_ERROR _UpdateFwFreelistSize(RGX_FREELIST *psFreeList,
 275                                                                                 IMG_BOOL bGrow,
 276                                                                                 IMG_UINT32 ui32DeltaSize)
 277 {
 278         PVRSRV_ERROR                    eError;
 279         RGXFWIF_KCCB_CMD                sGPCCBCmd;
 280
 281         sGPCCBCmd.eCmdType = (bGrow) ? RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE : RGXFWIF_KCCB_CMD_FREELIST_SHRINK_UPDATE;
 282         sGPCCBCmd.uCmdData.sFreeListGSData.psFreeListFWDevVAddr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
 283         sGPCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32DeltaSize;
 284         sGPCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = psFreeList->ui32CurrentFLPages;
 285
 286         PVR_DPF((PVR_DBG_MESSAGE, "Send FW update: freelist [FWAddr=0x%08x] has 0x%08x pages",
 287                                                                 psFreeList->sFreeListFWDevVAddr.ui32Addr,
 288                                                                 psFreeList->ui32CurrentFLPages));
 289
 290         /* Submit command to the firmware.  */
 291         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
 292         {
 293                 eError = RGXScheduleCommand(psFreeList->psDevInfo,
 294                                                                         RGXFWIF_DM_GP,
 295                                                                         &sGPCCBCmd,
 296                                                                         sizeof(sGPCCBCmd),
 297                                                                         IMG_TRUE);
 298                 if (eError != PVRSRV_ERROR_RETRY)
 299                 {
 300                         break;
 301                 }
 302                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
 303         } END_LOOP_UNTIL_TIMEOUT();
 304
 305         if (eError != PVRSRV_OK)
 306         {
 307                 PVR_DPF((PVR_DBG_ERROR, "_UpdateFwFreelistSize: failed to update FW freelist size. (error = %u)", eError));
 308                 return eError;
 309         }
 310
 311         return PVRSRV_OK;
 312 }
 313
 314 static PVRSRV_ERROR _FreeListCheckSum(RGX_FREELIST *psFreeList,
 315                                                            IMG_UINT64 *pui64CheckSum)
 316 {
 317 #if defined(NO_HARDWARE)
 318         /* No checksum needed as we have all information in the pdumps */
 319         PVR_UNREFERENCED_PARAMETER(psFreeList);
 320         *pui64CheckSum = 0;
 321         return PVRSRV_OK;
 322 #else
 323         PVRSRV_ERROR eError;
 324         IMG_SIZE_T uiNumBytes;
 325     IMG_UINT8* pui8Buffer;
 326     IMG_UINT32* pui32Buffer;
 327     IMG_UINT32 ui32CheckSumAdd = 0;
 328     IMG_UINT32 ui32CheckSumXor = 0;
 329     IMG_UINT32 ui32Entry;
 330     IMG_UINT32 ui32Entry2;
 331
 332         /* Allocate Buffer of the size of the freelist */
 333         pui8Buffer = OSAllocMem(psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
 334     if (pui8Buffer == IMG_NULL)
 335     {
 336                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 337         goto _OSAllocMem_Exit;
 338     }
 339
 340     /* Copy freelist content into Buffer */
 341     eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
 342                                 psFreeList->uiFreeListPMROffset + (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32),
 343                                 pui8Buffer,
 344                                 psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32),
 345                         &uiNumBytes);
 346     if (eError != PVRSRV_OK)
 347     {
 348         goto _PMR_ReadBytes_Exit;
 349     }
 350
 351     PVR_ASSERT(uiNumBytes == psFreeList->ui32CurrentFLPages * sizeof(IMG_UINT32));
 352
 353     /* Generate checksum */
 354     pui32Buffer = (IMG_UINT32 *)pui8Buffer;
 355     for(ui32Entry = 0; ui32Entry < psFreeList->ui32CurrentFLPages; ui32Entry++)
 356     {
 357         ui32CheckSumAdd += pui32Buffer[ui32Entry];
 358         ui32CheckSumXor ^= pui32Buffer[ui32Entry];
 359
 360         /* Check for double entries */
 361         for (ui32Entry2 = 0; ui32Entry2 < psFreeList->ui32CurrentFLPages; ui32Entry2++)
 362         {
 363                         if ((ui32Entry != ui32Entry2) &&
 364                                 (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2]))
 365                         {
 366                                 PVR_DPF((PVR_DBG_ERROR, "Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d",
 367                                                                                         psFreeList->sFreeListFWDevVAddr.ui32Addr,
 368                                                                                         pui32Buffer[ui32Entry2],
 369                                                                                         ui32Entry,
 370                                                                                         ui32Entry2));
 371                                 sleep_for_ever();
 372 //                              PVR_ASSERT(0);
 373                         }
 374         }
 375     }
 376
 377     OSFreeMem(pui8Buffer);
 378
 379     /* Set return value */
 380     *pui64CheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
 381     PVR_ASSERT(eError == PVRSRV_OK);
 382     return PVRSRV_OK;
 383
 384     /*
 385       error exit paths follow
 386     */
 387
 388 _PMR_ReadBytes_Exit:
 389         OSFreeMem(pui8Buffer);
 390
 391 _OSAllocMem_Exit:
 392     PVR_ASSERT(eError != PVRSRV_OK);
 393     return eError;
 394 #endif
 395 }
 396
 397 PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
 398                                                         IMG_UINT32 ui32NumPages,
 399                                                         PDLLIST_NODE pListHeader)
 400 {
 401         RGX_PMR_NODE    *psPMRNode;
 402         IMG_DEVMEM_SIZE_T uiSize;
 403         IMG_BOOL bMappingTable = IMG_TRUE;
 404         IMG_DEVMEM_OFFSET_T uiOffset;
 405         IMG_DEVMEM_SIZE_T uiLength;
 406         IMG_DEVMEM_SIZE_T uistartPage;
 407         PVRSRV_ERROR eError;
 408         IMG_UINT64 ui64CheckSum;
 409         IMG_UINT32 ui32CheckSumXor;
 410         IMG_UINT32 ui32CheckSumAdd;
 411
 412         /* Are we allowed to grow ? */
 413         if ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) < ui32NumPages)
 414         {
 415                 PVR_DPF((PVR_DBG_WARNING,"Freelist [0x%p]: grow by %u pages denied. Max PB size reached (current pages %u/%u)",
 416                                 psFreeList,
 417                                 ui32NumPages,
 418                                 psFreeList->ui32CurrentFLPages,
 419                                 psFreeList->ui32MaxFLPages));
 420                 return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
 421         }
 422
 423         /* Allocate kernel memory block structure */
 424         psPMRNode = OSAllocMem(sizeof(*psPMRNode));
 425         if (psPMRNode == IMG_NULL)
 426         {
 427                 PVR_DPF((PVR_DBG_ERROR, "RGXGrowFreeList: failed to allocate host data structure"));
 428                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 429                 goto ErrorAllocHost;
 430         }
 431
 432         /*
 433          * Lock protects simultaneous manipulation of:
 434          * - the memory block list
 435          * - the freelist's ui32CurrentFLPages
 436          */
 437         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
 438
 439
 440         psPMRNode->ui32NumPages = ui32NumPages;
 441         psPMRNode->psFreeList = psFreeList;
 442
 443         /* Allocate Memory Block */
 444         PDUMPCOMMENT("Allocate PB Block (Pages %08X)", ui32NumPages);
 445         uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
 446         eError = PhysmemNewRamBackedPMR(psFreeList->psDevInfo->psDeviceNode,
 447                                                                         uiSize,
 448                                                                         uiSize,
 449                                                                         1,
 450                                                                         1,
 451                                                                         &bMappingTable,
 452                                                                         RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 453                                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE,
 454                                                                         &psPMRNode->psPMR);
 455         if(eError != PVRSRV_OK)
 456         {
 457                 PVR_DPF((PVR_DBG_ERROR,
 458                                  "RGXGrowFreeList: Failed to allocate PB block of size: 0x%016llX",
 459                                  (IMG_UINT64)uiSize));
 460                 goto ErrorBlockAlloc;
 461         }
 462
 463         /* Zeroing physical pages pointed by the PMR */
 464         if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
 465         {
 466                 eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 467                 if (eError != PVRSRV_OK)
 468                 {
 469                         PVR_DPF((PVR_DBG_ERROR,"RGXGrowFreeList: Failed to zero PMR %p of freelist %p with Error %d",
 470                                                                         psPMRNode->psPMR,
 471                                                                         psFreeList,
 472                                                                         eError));
 473                         PVR_ASSERT(0);
 474                 }
 475         }
 476
 477         uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
 478         uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
 479         uiOffset = psFreeList->uiFreeListPMROffset + (uistartPage * sizeof(IMG_UINT32));
 480
 481         /* write Freelist with Memory Block physical addresses */
 482         eError = PMRWritePMPageList(
 483                                                 /* Target PMR, offset, and length */
 484                                                 psFreeList->psFreeListPMR,
 485                                                 uiOffset,
 486                                                 uiLength,
 487                                                 /* Referenced PMR, and "page" granularity */
 488                                                 psPMRNode->psPMR,
 489                                                 RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 490                                                 &psPMRNode->psPageList,
 491                                                 &ui64CheckSum);
 492         if (eError != PVRSRV_OK)
 493         {
 494                 PVR_DPF((PVR_DBG_ERROR,
 495                                  "RGXGrowFreeList: Failed to write pages of Node %p",
 496                                  psPMRNode));
 497                 goto ErrorPopulateFreelist;
 498         }
 499
 500         /* We add It must be added to the tail, otherwise the freelist population won't work */
 501         dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
 502
 503         /* Update number of available pages */
 504         psFreeList->ui32CurrentFLPages += ui32NumPages;
 505
 506         /* Update statistics */
 507         if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
 508         {
 509                 psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
 510         }
 511
 512         if (psFreeList->bCheckFreelist)
 513         {
 514                 /* Update checksum */
 515                 ui32CheckSumAdd = (IMG_UINT32)(psFreeList->ui64FreelistChecksum + ui64CheckSum);
 516                 ui32CheckSumXor = (IMG_UINT32)((psFreeList->ui64FreelistChecksum  ^ ui64CheckSum) >> 32);
 517                 psFreeList->ui64FreelistChecksum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
 518                 /* Note: We can't do a freelist check here, because the freelist is probably empty (OOM) */
 519         }
 520
 521         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 522
 523         PVR_DPF((PVR_DBG_MESSAGE,"Freelist [%p]: grow by %u pages (current pages %u/%u)",
 524                         psFreeList,
 525                         ui32NumPages,
 526                         psFreeList->ui32CurrentFLPages,
 527                         psFreeList->ui32MaxFLPages));
 528
 529         return PVRSRV_OK;
 530
 531         /* Error handling */
 532 ErrorPopulateFreelist:
 533         PMRUnrefPMR(psPMRNode->psPMR);
 534
 535 ErrorBlockAlloc:
 536         OSFreeMem(psPMRNode);
 537         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 538
 539 ErrorAllocHost:
 540         PVR_ASSERT(eError != PVRSRV_OK);
 541         return eError;
 542
 543 }
 544
 545 static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
 546                                                                                 RGX_FREELIST *psFreeList)
 547 {
 548         DLLIST_NODE *psNode;
 549         RGX_PMR_NODE *psPMRNode;
 550         PVRSRV_ERROR eError = PVRSRV_OK;
 551         IMG_UINT32 ui32OldValue;
 552
 553         /*
 554          * Lock protects simultaneous manipulation of:
 555          * - the memory block list
 556          * - the freelist's ui32CurrentFLPages value
 557          */
 558         PVR_ASSERT(pListHeader);
 559         PVR_ASSERT(psFreeList);
 560         PVR_ASSERT(psFreeList->psDevInfo);
 561         PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
 562
 563         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
 564
 565         /* Get node from head of list and remove it */
 566         psNode = dllist_get_next_node(pListHeader);
 567         if (psNode)
 568         {
 569                 dllist_remove_node(psNode);
 570
 571                 psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 572                 PVR_ASSERT(psPMRNode);
 573                 PVR_ASSERT(psPMRNode->psPMR);
 574                 PVR_ASSERT(psPMRNode->psFreeList);
 575
 576                 /* remove block from freelist list */
 577
 578                 /* Unwrite Freelist with Memory Block physical addresses */
 579                 eError = PMRUnwritePMPageList(psPMRNode->psPageList);
 580                 if (eError != PVRSRV_OK)
 581                 {
 582                         PVR_DPF((PVR_DBG_ERROR,
 583                                          "RGXRemoveBlockFromFreeListKM: Failed to unwrite pages of Node %p",
 584                                          psPMRNode));
 585                         PVR_ASSERT(IMG_FALSE);
 586                 }
 587
 588                 /* Free PMR (We should be the only one that holds a ref on the PMR) */
 589                 eError = PMRUnrefPMR(psPMRNode->psPMR);
 590                 if (eError != PVRSRV_OK)
 591                 {
 592                         PVR_DPF((PVR_DBG_ERROR,
 593                                          "RGXRemoveBlockFromFreeListKM: Failed to free PB block %p (error %u)",
 594                                          psPMRNode->psPMR,
 595                                          eError));
 596                         PVR_ASSERT(IMG_FALSE);
 597                 }
 598
 599                 /* update available pages in freelist */
 600                 ui32OldValue = psFreeList->ui32CurrentFLPages;
 601                 psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages;
 602
 603                 /* check underflow */
 604                 PVR_ASSERT(ui32OldValue > psFreeList->ui32CurrentFLPages);
 605
 606                 PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
 607                                                                 psFreeList,
 608                                                                 psPMRNode->ui32NumPages,
 609                                                                 psFreeList->ui32CurrentFLPages,
 610                                                                 psFreeList->ui32MaxFLPages));
 611
 612                 OSFreeMem(psPMRNode);
 613         }
 614         else
 615         {
 616                 PVR_DPF((PVR_DBG_WARNING,"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
 617                                                                 psFreeList,
 618                                                                 psFreeList->ui32InitFLPages));
 619                 eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
 620         }
 621
 622         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
 623
 624         return eError;
 625 }
 626
 627 static IMG_BOOL _FindFreeList(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
 628 {
 629         DEVMEM_FREELIST_LOOKUP *psRefLookUp = (DEVMEM_FREELIST_LOOKUP *)pvCallbackData;
 630         RGX_FREELIST *psFreeList;
 631
 632         psFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
 633
 634         if (psFreeList->ui32FreelistID == psRefLookUp->ui32FreeListID)
 635         {
 636                 psRefLookUp->psFreeList = psFreeList;
 637                 return IMG_FALSE;
 638         }
 639         else
 640         {
 641                 return IMG_TRUE;
 642         }
 643 }
 644
 645 IMG_VOID RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
 646                                                                 IMG_UINT32 ui32FreelistID)
 647 {
 648         DEVMEM_FREELIST_LOOKUP sLookUp;
 649         RGXFWIF_KCCB_CMD s3DCCBCmd;
 650         IMG_UINT32 ui32GrowValue;
 651         PVRSRV_ERROR eError;
 652
 653         PVR_ASSERT(psDevInfo);
 654
 655         /* find the freelist with the corresponding ID */
 656         sLookUp.ui32FreeListID = ui32FreelistID;
 657         sLookUp.psFreeList = IMG_NULL;
 658
 659         OSLockAcquire(psDevInfo->hLockFreeList);
 660         dllist_foreach_node(&psDevInfo->sFreeListHead, _FindFreeList, (IMG_PVOID)&sLookUp);
 661         OSLockRelease(psDevInfo->hLockFreeList);
 662
 663         if (sLookUp.psFreeList)
 664         {
 665                 RGX_FREELIST *psFreeList = sLookUp.psFreeList;
 666
 667                 /* Try to grow the freelist */
 668                 eError = RGXGrowFreeList(psFreeList,
 669                                                                 psFreeList->ui32GrowFLPages,
 670                                                                 &psFreeList->sMemoryBlockHead);
 671                 if (eError == PVRSRV_OK)
 672                 {
 673                         /* Grow successful, return size of grow size */
 674                         ui32GrowValue = psFreeList->ui32GrowFLPages;
 675
 676                         psFreeList->ui32NumGrowReqByFW++;
 677
 678  #if defined(PVRSRV_ENABLE_PROCESS_STATS)
 679                         /* Update Stats */
 680                         PVRSRVStatsUpdateFreelistStats(0,
 681                                        1, /* Add 1 to the appropriate counter (Requests by FW) */
 682                                        psFreeList->ui32InitFLPages,
 683                                        psFreeList->ui32NumHighPages,
 684                                        psFreeList->ownerPid);
 685
 686  #endif
 687
 688                 }
 689                 else
 690                 {
 691                         /* Grow failed */
 692                         ui32GrowValue = 0;
 693                         PVR_DPF((PVR_DBG_ERROR,"Grow for FreeList %p failed (error %u)",
 694                                                                         psFreeList,
 695                                                                         eError));
 696                 }
 697
 698                 /* send feedback */
 699                 s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
 700                 s3DCCBCmd.uCmdData.sFreeListGSData.psFreeListFWDevVAddr = sLookUp.psFreeList->sFreeListFWDevVAddr.ui32Addr;
 701                 s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaSize = ui32GrowValue;
 702                 s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewSize = psFreeList->ui32CurrentFLPages;
 703
 704                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
 705                 {
 706                         eError = RGXScheduleCommand(psDevInfo,
 707                                                                                                 RGXFWIF_DM_3D,
 708                                                                                                 &s3DCCBCmd,
 709                                                                                                 sizeof(s3DCCBCmd),
 710                                                                                                 IMG_FALSE);
 711                         if (eError != PVRSRV_ERROR_RETRY)
 712                         {
 713                                 break;
 714                         }
 715                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
 716                 } END_LOOP_UNTIL_TIMEOUT();
 717                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
 718
 719                 PVR_ASSERT(eError == PVRSRV_OK);
 720         }
 721         else
 722         {
 723                 /* Should never happen */
 724                 PVR_DPF((PVR_DBG_ERROR,"FreeList Lookup for FreeList ID 0x%08x failed (Populate)", sLookUp.ui32FreeListID));
 725                 PVR_ASSERT(IMG_FALSE);
 726         }
 727 }
 728
 729 static IMG_BOOL _RGXCheckFreeListReconstruction(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
 730 {
 731
 732         PVRSRV_RGXDEV_INFO              *psDevInfo;
 733         RGX_FREELIST                    *psFreeList;
 734         RGX_PMR_NODE                    *psPMRNode;
 735         PVRSRV_ERROR                    eError;
 736         IMG_DEVMEM_OFFSET_T             uiOffset;
 737         IMG_DEVMEM_SIZE_T               uiLength;
 738         IMG_UINT32                              ui32StartPage;
 739         IMG_UINT64                              ui64CheckSum;
 740
 741         psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
 742         psFreeList = psPMRNode->psFreeList;
 743         PVR_ASSERT(psFreeList);
 744         psDevInfo = psFreeList->psDevInfo;
 745         PVR_ASSERT(psDevInfo);
 746
 747         uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
 748         ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
 749         uiOffset = psFreeList->uiFreeListPMROffset + (ui32StartPage * sizeof(IMG_UINT32));
 750
 751         PMRUnwritePMPageList(psPMRNode->psPageList);
 752         psPMRNode->psPageList = IMG_NULL;
 753         eError = PMRWritePMPageList(
 754                                                 /* Target PMR, offset, and length */
 755                                                 psFreeList->psFreeListPMR,
 756                                                 uiOffset,
 757                                                 uiLength,
 758                                                 /* Referenced PMR, and "page" granularity */
 759                                                 psPMRNode->psPMR,
 760                                                 RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
 761                                                 &psPMRNode->psPageList,
 762                                                 &ui64CheckSum);
 763
 764         if (eError != PVRSRV_OK)
 765         {
 766                 PVR_DPF((PVR_DBG_ERROR,"Error (%u) writing FL 0x%08x", eError, (IMG_UINT32)psFreeList->ui32FreelistID));
 767         }
 768
 769         /* Zeroing physical pages pointed by the reconstructed freelist */
 770         if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
 771         {
 772                 eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
 773                 if (eError != PVRSRV_OK)
 774                 {
 775                         PVR_DPF((PVR_DBG_ERROR,"_RGXCheckFreeListReconstruction: Failed to zero PMR %p of freelist %p with Error %d",
 776                                                                         psPMRNode->psPMR,
 777                                                                         psFreeList,
 778                                                                         eError));
 779                         PVR_ASSERT(0);
 780                 }
 781         }
 782
 783         psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
 784
 785         return IMG_TRUE;
 786 }
 787
 788 IMG_VOID RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
 789                                                                 RGXFWIF_DM eDM,
 790                                                                 IMG_UINT32 ui32FreelistsCount,
 791                                                                 IMG_UINT32 *paui32Freelists)
 792 {
 793         PVRSRV_ERROR eError;
 794         DEVMEM_FREELIST_LOOKUP sLookUp;
 795         IMG_UINT32 ui32Loop, ui32Loop2;
 796         RGXFWIF_KCCB_CMD s3DCCBCmd;
 797         IMG_UINT64 ui64CheckSum;
 798
 799         PVR_ASSERT(psDevInfo);
 800
 801         //PVR_DPF((PVR_DBG_ERROR,"FreeList RECONSTRUCTION: Reconstructing %u freelist(s)", ui32FreelistsCount));
 802
 803         for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
 804         {
 805                 /* check if there is more than one occurrence of FL on the list */
 806                 for (ui32Loop2 = ui32Loop + 1; ui32Loop2 < ui32FreelistsCount; ui32Loop2++)
 807                 {
 808                         if (paui32Freelists[ui32Loop] == paui32Freelists[ui32Loop2])
 809                         {
 810                                 /* There is a duplicate on a list, skip current Freelist */
 811                                 break;
 812                         }
 813                 }
 814
 815                 if (ui32Loop2 < ui32FreelistsCount)
 816                 {
 817                         /* There is a duplicate on the list, skip current Freelist */
 818                         continue;
 819                 }
 820
 821                 /* find the freelist with the corresponding ID */
 822                 sLookUp.ui32FreeListID = paui32Freelists[ui32Loop];
 823                 sLookUp.psFreeList = IMG_NULL;
 824
 825                 //PVR_DPF((PVR_DBG_ERROR,"FreeList RECONSTRUCTION: Looking for freelist %08X", (IMG_UINT32)sLookUp.ui32FreeListID));
 826                 OSLockAcquire(psDevInfo->hLockFreeList);
 827                 //PVR_DPF((PVR_DBG_ERROR,"FreeList RECONSTRUCTION: Freelist head %08X", (IMG_UINT32)&psDevInfo->sFreeListHead));
 828                 dllist_foreach_node(&psDevInfo->sFreeListHead, _FindFreeList, (IMG_PVOID)&sLookUp);
 829                 OSLockRelease(psDevInfo->hLockFreeList);
 830
 831                 if (sLookUp.psFreeList)
 832                 {
 833                         RGX_FREELIST *psFreeList = sLookUp.psFreeList;
 834
 835                         //PVR_DPF((PVR_DBG_ERROR,"FreeList RECONSTRUCTION: Reconstructing freelist %08X", (IMG_UINT32)psFreeList));
 836
 837                         /* Do the FreeList Reconstruction */
 838
 839                         psFreeList->ui32CurrentFLPages = 0;
 840
 841                         /* Reconstructing Init FreeList pages */
 842                         dllist_foreach_node(&psFreeList->sMemoryBlockInitHead,
 843                                                         _RGXCheckFreeListReconstruction,
 844                                                         IMG_NULL);
 845
 846                         /* Reconstructing Grow FreeList pages */
 847                         dllist_foreach_node(&psFreeList->sMemoryBlockHead,
 848                                                         _RGXCheckFreeListReconstruction,
 849                                                         IMG_NULL);
 850
 851                         if (psFreeList->bCheckFreelist)
 852                         {
 853                                 /* Get Freelist checksum (as the list is fully populated) */
 854                                 eError = _FreeListCheckSum(psFreeList,
 855                                                                                         &ui64CheckSum);
 856                                 if (eError != PVRSRV_OK)
 857                                 {
 858                                         PVR_DPF((PVR_DBG_ERROR,
 859                                                          "RGXProcessRequestFreelistsReconstruction: Failed to get freelist checksum Node %p",
 860                                                          psFreeList));
 861                                         sleep_for_ever();
 862 //                                      PVR_ASSERT(0);
 863                                 }
 864
 865                                 /* Verify checksum with previous value */
 866                                 if (psFreeList->ui64FreelistChecksum != ui64CheckSum)
 867                                 {
 868                                         PVR_DPF((PVR_DBG_ERROR, "RGXProcessRequestFreelistsReconstruction: Freelist [%p] checksum failed: before reconstruction = 0x%016llx, after reconstruction = 0x%016llx",
 869                                                                                         psFreeList,
 870                                                                                         psFreeList->ui64FreelistChecksum,
 871                                                                                         ui64CheckSum));
 872                                         sleep_for_ever();
 873                                         //PVR_ASSERT(0);
 874                                 }
 875                         }
 876
 877                         eError = PVRSRV_OK;
 878
 879                         if (eError == PVRSRV_OK)
 880                         {
 881                                 /* Freelist reconstruction successful */
 882                                 s3DCCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] =
 883                                                                                                         paui32Freelists[ui32Loop];
 884                         }
 885                         else
 886                         {
 887                                 /* Freelist reconstruction failed */
 888                                 s3DCCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] =
 889                                                                                                         paui32Freelists[ui32Loop] | RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
 890
 891                                 PVR_DPF((PVR_DBG_ERROR,"Reconstructing of FreeList %p failed (error %u)",
 892                                                                                 psFreeList,
 893                                                                                 eError));
 894                         }
 895                 }
 896                 else
 897                 {
 898                         /* Should never happen */
 899                         PVR_DPF((PVR_DBG_ERROR,"FreeList Lookup for FreeList ID 0x%08x failed (Freelist reconstruction)", sLookUp.ui32FreeListID));
 900                         PVR_ASSERT(IMG_FALSE);
 901                 }
 902         }
 903
 904         /* send feedback */
 905         s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
 906         s3DCCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
 907
 908         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
 909         {
 910                 eError = RGXScheduleCommand(psDevInfo,
 911                                                                                         eDM,
 912                                                                                         &s3DCCBCmd,
 913                                                                                         sizeof(s3DCCBCmd),
 914                                                                                         IMG_FALSE);
 915                 if (eError != PVRSRV_ERROR_RETRY)
 916                 {
 917                         break;
 918                 }
 919                 OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
 920         } END_LOOP_UNTIL_TIMEOUT();
 921
 922         /* Kernel CCB should never fill up, as the FW is processing them right away  */
 923         PVR_ASSERT(eError == PVRSRV_OK);
 924 }
 925
 926 /* Create HWRTDataSet */
 927 IMG_EXPORT
 928 PVRSRV_ERROR RGXCreateHWRTData(PVRSRV_DEVICE_NODE       *psDeviceNode,
 929                                                            IMG_UINT32                   psRenderTarget,
 930                                                            IMG_DEV_VIRTADDR             psPMMListDevVAddr,
 931                                                            IMG_DEV_VIRTADDR             psVFPPageTableAddr,
 932                                                            RGX_FREELIST                 *apsFreeLists[RGXFW_MAX_FREELISTS],
 933                                                            RGX_RTDATA_CLEANUP_DATA      **ppsCleanupData,
 934                                                            DEVMEM_MEMDESC               **ppsRTACtlMemDesc,
 935                                                            IMG_UINT32           ui32PPPScreen,
 936                                                            IMG_UINT32           ui32PPPGridOffset,
 937                                                            IMG_UINT64           ui64PPPMultiSampleCtl,
 938                                                            IMG_UINT32           ui32TPCStride,
 939                                                            IMG_DEV_VIRTADDR             sTailPtrsDevVAddr,
 940                                                            IMG_UINT32           ui32TPCSize,
 941                                                            IMG_UINT32           ui32TEScreen,
 942                                                            IMG_UINT32           ui32TEAA,
 943                                                            IMG_UINT32           ui32TEMTILE1,
 944                                                            IMG_UINT32           ui32TEMTILE2,
 945                                                            IMG_UINT32           ui32MTileStride,
 946                                                            IMG_UINT32                 ui32ISPMergeLowerX,
 947                                                            IMG_UINT32                 ui32ISPMergeLowerY,
 948                                                            IMG_UINT32                 ui32ISPMergeUpperX,
 949                                                            IMG_UINT32                 ui32ISPMergeUpperY,
 950                                                            IMG_UINT32                 ui32ISPMergeScaleX,
 951                                                            IMG_UINT32                 ui32ISPMergeScaleY,
 952                                                            IMG_UINT16                   ui16MaxRTs,
 953                                                            DEVMEM_MEMDESC               **ppsMemDesc,
 954                                                            IMG_UINT32                   *puiHWRTData)
 955 {
 956         PVRSRV_ERROR eError;
 957         PVRSRV_RGXDEV_INFO *psDevInfo;
 958         RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
 959         RGXFWIF_HWRTDATA *psHWRTData;
 960         RGXFWIF_RTA_CTL *psRTACtl;
 961         IMG_UINT32 ui32Loop;
 962         RGX_RTDATA_CLEANUP_DATA *psTmpCleanup;
 963
 964         /* Prepare cleanup struct */
 965         psTmpCleanup = OSAllocMem(sizeof(*psTmpCleanup));
 966         if (psTmpCleanup == IMG_NULL)
 967         {
 968                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 969                 goto AllocError;
 970         }
 971
 972         OSMemSet(psTmpCleanup, 0, sizeof(*psTmpCleanup));
 973         *ppsCleanupData = psTmpCleanup;
 974
 975         /* Allocate cleanup sync */
 976         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
 977                                                    &psTmpCleanup->psCleanupSync,
 978                                                    "HWRTData cleanup");
 979         if (eError != PVRSRV_OK)
 980         {
 981                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate cleanup sync (0x%x)",
 982                                 eError));
 983                 goto SyncAlloc;
 984         }
 985
 986         psDevInfo = psDeviceNode->pvDevice;
 987
 988         /*
 989          * This FW RT-Data is only mapped into kernel for initialisation.
 990          * Otherwise this allocation is only used by the FW.
 991          * Therefore the GPU cache doesn't need coherency,
 992          * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
 993          */
 994         eError = DevmemFwAllocate(psDevInfo,
 995                                                         sizeof(RGXFWIF_HWRTDATA),
 996                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
 997                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) |
 998                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
 999                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1000                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1001                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1002                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1003                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1004                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1005                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1006                                                         "FirmwareHWRTData",
1007                                                         ppsMemDesc);
1008         if (eError != PVRSRV_OK)
1009         {
1010                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateHWRTData: DevmemAllocate for RGX_FWIF_HWRTDATA failed"));
1011                 goto FWRTDataAllocateError;
1012         }
1013
1014         psTmpCleanup->psDeviceNode = psDeviceNode;
1015         psTmpCleanup->psFWHWRTDataMemDesc = *ppsMemDesc;
1016
1017         RGXSetFirmwareAddress(&pFirmwareAddr, *ppsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1018
1019         *puiHWRTData = pFirmwareAddr.ui32Addr;
1020
1021         eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, (IMG_VOID **)&psHWRTData);
1022         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
1023
1024
1025         psHWRTData->psPMMListDevVAddr = psPMMListDevVAddr;
1026         psHWRTData->psParentRenderTarget.ui32Addr = psRenderTarget;
1027         #if defined(SUPPORT_VFP)
1028         psHWRTData->sVFPPageTableAddr = psVFPPageTableAddr;
1029         #endif
1030
1031         psHWRTData->ui32PPPScreen         = ui32PPPScreen;
1032         psHWRTData->ui32PPPGridOffset     = ui32PPPGridOffset;
1033         psHWRTData->ui64PPPMultiSampleCtl = ui64PPPMultiSampleCtl;
1034         psHWRTData->ui32TPCStride         = ui32TPCStride;
1035         psHWRTData->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
1036         psHWRTData->ui32TPCSize           = ui32TPCSize;
1037         psHWRTData->ui32TEScreen          = ui32TEScreen;
1038         psHWRTData->ui32TEAA              = ui32TEAA;
1039         psHWRTData->ui32TEMTILE1          = ui32TEMTILE1;
1040         psHWRTData->ui32TEMTILE2          = ui32TEMTILE2;
1041         psHWRTData->ui32MTileStride       = ui32MTileStride;
1042         psHWRTData->ui32ISPMergeLowerX = ui32ISPMergeLowerX;
1043         psHWRTData->ui32ISPMergeLowerY = ui32ISPMergeLowerY;
1044         psHWRTData->ui32ISPMergeUpperX = ui32ISPMergeUpperX;
1045         psHWRTData->ui32ISPMergeUpperY = ui32ISPMergeUpperY;
1046         psHWRTData->ui32ISPMergeScaleX = ui32ISPMergeScaleX;
1047         psHWRTData->ui32ISPMergeScaleY = ui32ISPMergeScaleY;
1048
1049         OSLockAcquire(psDevInfo->hLockFreeList);
1050         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1051         {
1052                 psTmpCleanup->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
1053                 psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount++;
1054                 psHWRTData->apsFreeLists[ui32Loop] = *((PRGXFWIF_FREELIST *)&(psTmpCleanup->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr));
1055                 /* invalid initial snapshot value, the snapshot is always taken during first kick
1056                  * and hence the value get replaced during the first kick anyway. So its safe to set it 0.
1057                 */
1058                 psHWRTData->aui32FreeListHWRSnapshot[ui32Loop] = 0;
1059         }
1060         OSLockRelease(psDevInfo->hLockFreeList);
1061
1062         PDUMPCOMMENT("Allocate RGXFW RTA control");
1063         eError = DevmemFwAllocate(psDevInfo,
1064                                                                                 sizeof(RGXFWIF_RTA_CTL),
1065                                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1066                                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) |
1067                                                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1068                                                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1069                                                                                 PVRSRV_MEMALLOCFLAG_UNCACHED |
1070                                                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
1071                                                                                 "FirmwareRTAControl",
1072                                                                                 ppsRTACtlMemDesc);
1073
1074         if (eError != PVRSRV_OK)
1075         {
1076                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate RGX RTA control (%u)",
1077                                 eError));
1078                 goto FWRTAAllocateError;
1079         }
1080         psTmpCleanup->psRTACtlMemDesc = *ppsRTACtlMemDesc;
1081         RGXSetFirmwareAddress(&psHWRTData->psRTACtl,
1082                                                                    *ppsRTACtlMemDesc,
1083                                                                    0, RFW_FWADDR_FLAG_NONE);
1084
1085         eError = DevmemAcquireCpuVirtAddr(*ppsRTACtlMemDesc, (IMG_VOID **)&psRTACtl);
1086         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTACpuMapError);
1087         psRTACtl->ui32RenderTargetIndex = 0;
1088         psRTACtl->ui32ActiveRenderTargets = 0;
1089
1090         if (ui16MaxRTs > 1)
1091         {
1092                 /* Allocate memory for the checks */
1093                 PDUMPCOMMENT("Allocate memory for shadow render target cache");
1094                 eError = DevmemFwAllocate(psDevInfo,
1095                                                                 ui16MaxRTs,
1096                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1097                                                                 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(META_CACHED) |
1098                                                                 PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1099                                                                 PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1100                                                                 PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1101                                                                 PVRSRV_MEMALLOCFLAG_UNCACHED|
1102                                                                 PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
1103                                                                 "FirmwareShadowRTCache",
1104                                                                 &psTmpCleanup->psRTArrayMemDesc);
1105                 if (eError != PVRSRV_OK)
1106                 {
1107                         PVR_DPF((PVR_DBG_ERROR,"RGXCreateHWRTData: Failed to allocate %d bytes for render target array (%u)",
1108                                                 ui16MaxRTs,
1109                                                 eError));
1110                         goto FWAllocateRTArryError;
1111                 }
1112
1113                 RGXSetFirmwareAddress(&psRTACtl->paui32ValidRenderTargets,
1114                                                                                 psTmpCleanup->psRTArrayMemDesc,
1115                                                                                 0, RFW_FWADDR_FLAG_NONE);
1116         }
1117         else
1118         {
1119                 psRTACtl->paui32ValidRenderTargets.ui32Addr = 0;
1120         }
1121
1122         PDUMPCOMMENT("Dump HWRTData 0x%08X", *puiHWRTData);
1123         DevmemPDumpLoadMem(*ppsMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
1124         PDUMPCOMMENT("Dump RTACtl");
1125         DevmemPDumpLoadMem(*ppsRTACtlMemDesc, 0, sizeof(*psRTACtl), PDUMP_FLAGS_CONTINUOUS);
1126
1127         DevmemReleaseCpuVirtAddr(*ppsMemDesc);
1128         DevmemReleaseCpuVirtAddr(*ppsRTACtlMemDesc);
1129         return PVRSRV_OK;
1130
1131         DevmemFwFree(psTmpCleanup->psRTArrayMemDesc);
1132 FWAllocateRTArryError:
1133         DevmemReleaseCpuVirtAddr(*ppsRTACtlMemDesc);
1134 FWRTACpuMapError:
1135         RGXUnsetFirmwareAddress(*ppsRTACtlMemDesc);
1136         DevmemFwFree(*ppsRTACtlMemDesc);
1137 FWRTAAllocateError:
1138         OSLockAcquire(psDevInfo->hLockFreeList);
1139         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1140         {
1141                 PVR_ASSERT(psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount > 0);
1142                 psTmpCleanup->apsFreeLists[ui32Loop]->ui32RefCount--;
1143         }
1144         OSLockRelease(psDevInfo->hLockFreeList);
1145         DevmemReleaseCpuVirtAddr(*ppsMemDesc);
1146 FWRTDataCpuMapError:
1147         RGXUnsetFirmwareAddress(*ppsMemDesc);
1148         DevmemFwFree(*ppsMemDesc);
1149 FWRTDataAllocateError:
1150         SyncPrimFree(psTmpCleanup->psCleanupSync);
1151 SyncAlloc:
1152         OSFreeMem(psTmpCleanup);
1153
1154 AllocError:
1155         return eError;
1156 }
1157
1158 /* Destroy HWRTDataSet */
1159 IMG_EXPORT
1160 PVRSRV_ERROR RGXDestroyHWRTData(RGX_RTDATA_CLEANUP_DATA *psCleanupData)
1161 {
1162         PVRSRV_RGXDEV_INFO *psDevInfo;
1163         PVRSRV_ERROR eError;
1164         PRGXFWIF_HWRTDATA psHWRTData;
1165         IMG_UINT32 ui32Loop;
1166
1167         PVR_ASSERT(psCleanupData);
1168
1169         RGXSetFirmwareAddress(&psHWRTData, psCleanupData->psFWHWRTDataMemDesc, 0, RFW_FWADDR_NOREF_FLAG);
1170
1171         /* Cleanup HWRTData in TA */
1172         eError = RGXFWRequestHWRTDataCleanUp(psCleanupData->psDeviceNode,
1173                                                                                  psHWRTData,
1174                                                                                  psCleanupData->psCleanupSync,
1175                                                                                  RGXFWIF_DM_TA);
1176         if (eError == PVRSRV_ERROR_RETRY)
1177         {
1178                 return eError;
1179         }
1180
1181         psDevInfo = psCleanupData->psDeviceNode->pvDevice;
1182
1183         /* Cleanup HWRTData in 3D */
1184         eError = RGXFWRequestHWRTDataCleanUp(psCleanupData->psDeviceNode,
1185                                                                                  psHWRTData,
1186                                                                                  psCleanupData->psCleanupSync,
1187                                                                                  RGXFWIF_DM_3D);
1188         if (eError == PVRSRV_ERROR_RETRY)
1189         {
1190                 return eError;
1191         }
1192
1193         /* If we got here then TA and 3D operations on this RTData have finished */
1194         if(psCleanupData->psRTACtlMemDesc)
1195         {
1196                 RGXUnsetFirmwareAddress(psCleanupData->psRTACtlMemDesc);
1197                 DevmemFwFree(psCleanupData->psRTACtlMemDesc);
1198         }
1199
1200         RGXUnsetFirmwareAddress(psCleanupData->psFWHWRTDataMemDesc);
1201         DevmemFwFree(psCleanupData->psFWHWRTDataMemDesc);
1202
1203         if(psCleanupData->psRTArrayMemDesc)
1204         {
1205                 RGXUnsetFirmwareAddress(psCleanupData->psRTArrayMemDesc);
1206                 DevmemFwFree(psCleanupData->psRTArrayMemDesc);
1207         }
1208
1209         SyncPrimFree(psCleanupData->psCleanupSync);
1210
1211         /* decrease freelist refcount */
1212         OSLockAcquire(psDevInfo->hLockFreeList);
1213         for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
1214         {
1215                 PVR_ASSERT(psCleanupData->apsFreeLists[ui32Loop]->ui32RefCount > 0);
1216                 psCleanupData->apsFreeLists[ui32Loop]->ui32RefCount--;
1217         }
1218         OSLockRelease(psDevInfo->hLockFreeList);
1219
1220         OSFreeMem(psCleanupData);
1221
1222         return PVRSRV_OK;
1223 }
1224
1225 IMG_EXPORT
1226 PVRSRV_ERROR RGXCreateFreeList(PVRSRV_DEVICE_NODE       *psDeviceNode,
1227                                                            IMG_UINT32                   ui32MaxFLPages,
1228                                                            IMG_UINT32                   ui32InitFLPages,
1229                                                            IMG_UINT32                   ui32GrowFLPages,
1230                                                            IMG_BOOL                             bCheckFreelist,
1231                                                            IMG_DEV_VIRTADDR             sFreeListDevVAddr,
1232                                                            PMR                                  *psFreeListPMR,
1233                                                            IMG_DEVMEM_OFFSET_T  uiFreeListPMROffset,
1234                                                            RGX_FREELIST                 **ppsFreeList)
1235 {
1236         PVRSRV_ERROR                            eError;
1237         RGXFWIF_FREELIST                        *psFWFreeList;
1238         DEVMEM_MEMDESC                          *psFWFreelistMemDesc;
1239         RGX_FREELIST                            *psFreeList;
1240         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
1241
1242         /* Allocate kernel freelist struct */
1243         psFreeList = OSAllocMem(sizeof(*psFreeList));
1244         if (psFreeList == IMG_NULL)
1245         {
1246                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateFreeList: failed to allocate host data structure"));
1247                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1248                 goto ErrorAllocHost;
1249         }
1250         OSMemSet(psFreeList, 0, sizeof(*psFreeList));
1251
1252         /* Allocate cleanup sync */
1253         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
1254                                                    &psFreeList->psCleanupSync,
1255                                                    "ta3d free list cleanup");
1256         if (eError != PVRSRV_OK)
1257         {
1258                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateFreeList: Failed to allocate cleanup sync (0x%x)",
1259                                 eError));
1260                 goto SyncAlloc;
1261         }
1262
1263         /*
1264          * This FW FreeList context is only mapped into kernel for initialisation.
1265          * Otherwise this allocation is only used by the FW.
1266          * Therefore the GPU cache doesn't need coherency,
1267          * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
1268          */
1269         eError = DevmemFwAllocate(psDevInfo,
1270                                                         sizeof(*psFWFreeList),
1271                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1272                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1273                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1274                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1275                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1276                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1277                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1278                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1279                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1280                                                         "FirmwareFreeList",
1281                                                         &psFWFreelistMemDesc);
1282         if (eError != PVRSRV_OK)
1283         {
1284                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateFreeList: DevmemAllocate for RGXFWIF_FREELIST failed"));
1285                 goto FWFreeListAlloc;
1286         }
1287
1288         /* Initialise host data structures */
1289         psFreeList->psDevInfo = psDevInfo;
1290         psFreeList->psFreeListPMR = psFreeListPMR;
1291         psFreeList->uiFreeListPMROffset = uiFreeListPMROffset;
1292         psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
1293         RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1294         psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
1295         psFreeList->ui32MaxFLPages = ui32MaxFLPages;
1296         psFreeList->ui32InitFLPages = ui32InitFLPages;
1297         psFreeList->ui32GrowFLPages = ui32GrowFLPages;
1298         psFreeList->ui32CurrentFLPages = 0;
1299         psFreeList->ui64FreelistChecksum = 0;
1300         psFreeList->ui32RefCount = 0;
1301         psFreeList->bCheckFreelist = bCheckFreelist;
1302         dllist_init(&psFreeList->sMemoryBlockHead);
1303         dllist_init(&psFreeList->sMemoryBlockInitHead);
1304
1305
1306         /* Add to list of freelists */
1307         OSLockAcquire(psDevInfo->hLockFreeList);
1308         dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
1309         OSLockRelease(psDevInfo->hLockFreeList);
1310
1311
1312         /* Initialise FW data structure */
1313         eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (IMG_VOID **)&psFWFreeList);
1314         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
1315         psFWFreeList->ui32MaxPages = ui32MaxFLPages;
1316         psFWFreeList->ui32CurrentPages = ui32InitFLPages;
1317         psFWFreeList->ui32GrowPages = ui32GrowFLPages;
1318         psFWFreeList->ui64CurrentStackTop = ui32InitFLPages - 1;
1319         psFWFreeList->psFreeListDevVAddr = sFreeListDevVAddr;
1320         psFWFreeList->ui64CurrentDevVAddr = sFreeListDevVAddr.uiAddr + ((ui32MaxFLPages - ui32InitFLPages) * sizeof(IMG_UINT32));
1321         psFWFreeList->ui32FreeListID = psFreeList->ui32FreelistID;
1322         psFWFreeList->bGrowPending = IMG_FALSE;
1323
1324         PVR_DPF((PVR_DBG_MESSAGE,"Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, Max FL base address 0x%016llx, Init FL base address 0x%016llx",
1325                         psFreeList,
1326                         ui32MaxFLPages,
1327                         ui32InitFLPages,
1328                         sFreeListDevVAddr.uiAddr,
1329                         psFWFreeList->psFreeListDevVAddr.uiAddr));
1330
1331         PDUMPCOMMENT("Dump FW FreeList");
1332         DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
1333
1334         /*
1335          * Separate dump of the Freelist's number of Pages and stack pointer.
1336          * This allows to easily modify the PB size in the out2.txt files.
1337          */
1338         PDUMPCOMMENT("FreeList TotalPages");
1339         DevmemPDumpLoadMemValue64(psFreeList->psFWFreelistMemDesc,
1340                                                         offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
1341                                                         psFWFreeList->ui32CurrentPages,
1342                                                         PDUMP_FLAGS_CONTINUOUS);
1343         PDUMPCOMMENT("FreeList StackPointer");
1344         DevmemPDumpLoadMemValue64(psFreeList->psFWFreelistMemDesc,
1345                                                         offsetof(RGXFWIF_FREELIST, ui64CurrentStackTop),
1346                                                         psFWFreeList->ui64CurrentStackTop,
1347                                                         PDUMP_FLAGS_CONTINUOUS);
1348
1349         DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
1350
1351
1352         /* Add initial PB block */
1353         eError = RGXGrowFreeList(psFreeList,
1354                                                                 ui32InitFLPages,
1355                                                                 &psFreeList->sMemoryBlockInitHead);
1356         if (eError != PVRSRV_OK)
1357         {
1358                 PVR_DPF((PVR_DBG_ERROR,
1359                                 "RGXCreateFreeList: failed to allocate initial memory block for free list 0x%016llx (error = %u)",
1360                                 sFreeListDevVAddr.uiAddr,
1361                                 eError));
1362                 goto ErrorAllocBlock;
1363         }
1364 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
1365                         /* Update Stats */
1366                         PVRSRVStatsUpdateFreelistStats(1, /* Add 1 to the appropriate counter (Requests by App)*/
1367                                        0,
1368                                        psFreeList->ui32InitFLPages,
1369                                        psFreeList->ui32NumHighPages,
1370                                        psFreeList->ownerPid);
1371
1372 #endif
1373
1374         psFreeList->ownerPid = OSGetCurrentProcessIDKM();
1375         /* return values */
1376         *ppsFreeList = psFreeList;
1377
1378         return PVRSRV_OK;
1379
1380         /* Error handling */
1381
1382 ErrorAllocBlock:
1383         DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
1384
1385 FWFreeListCpuMap:
1386         /* Remove freelists from list  */
1387         OSLockAcquire(psDevInfo->hLockFreeList);
1388         dllist_remove_node(&psFreeList->sNode);
1389         OSLockRelease(psDevInfo->hLockFreeList);
1390
1391         RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
1392         DevmemFwFree(psFWFreelistMemDesc);
1393
1394 FWFreeListAlloc:
1395         SyncPrimFree(psFreeList->psCleanupSync);
1396
1397 SyncAlloc:
1398         OSFreeMem(psFreeList);
1399
1400 ErrorAllocHost:
1401         PVR_ASSERT(eError != PVRSRV_OK);
1402         return eError;
1403 }
1404
1405
1406 /*
1407         RGXDestroyFreeList
1408 */
1409 IMG_EXPORT
1410 PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
1411 {
1412         PVRSRV_ERROR eError;
1413         IMG_UINT64 ui64CheckSum;
1414
1415         PVR_ASSERT(psFreeList);
1416
1417         if (psFreeList->ui32RefCount != 0)
1418         {
1419                 /* Freelist still busy */
1420                 return PVRSRV_ERROR_RETRY;
1421         }
1422
1423         /* Freelist is not in use => start firmware cleanup */
1424         eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
1425                                                                                  psFreeList->sFreeListFWDevVAddr,
1426                                                                                  psFreeList->psCleanupSync);
1427         if(eError != PVRSRV_OK)
1428         {
1429                 /* Can happen if the firmware took too long to handle the cleanup request,
1430                  * or if SLC-flushes didn't went through (due to some GPU lockup) */
1431                 return eError;
1432         }
1433
1434         if (psFreeList->bCheckFreelist)
1435         {
1436                 /* Do consistency tests (as the list is fully populated) */
1437                 eError = _FreeListCheckSum(psFreeList,
1438                                                                         &ui64CheckSum);
1439                 if (eError != PVRSRV_OK)
1440                 {
1441                         PVR_DPF((PVR_DBG_ERROR,
1442                                          "RGXDestroyFreeList: Failed to get freelist checksum Node %p",
1443                                          psFreeList));
1444                         sleep_for_ever();
1445 //                              PVR_ASSERT(0);
1446                 }
1447
1448                 if (psFreeList->ui64FreelistChecksum != ui64CheckSum)
1449                 {
1450                         PVR_DPF((PVR_DBG_ERROR,
1451                                          "RGXDestroyFreeList: Checksum mismatch [%p]! stored 0x%016llx, verified 0x%016llx %p",
1452                                          psFreeList,
1453                                          psFreeList->ui64FreelistChecksum,
1454                                          ui64CheckSum,
1455                                          psFreeList));
1456                         sleep_for_ever();
1457 //                      PVR_ASSERT(0);
1458                 }
1459         }
1460
1461         /* Destroy FW structures */
1462         RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
1463         DevmemFwFree(psFreeList->psFWFreelistMemDesc);
1464
1465         /* Remove grow shrink blocks */
1466         while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
1467         {
1468                 eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
1469                 PVR_ASSERT(eError == PVRSRV_OK);
1470         }
1471
1472         /* Remove initial PB block */
1473         eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
1474         PVR_ASSERT(eError == PVRSRV_OK);
1475
1476         /* consistency checks */
1477         PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
1478         PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
1479
1480         /* Remove FreeList from list */
1481         OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
1482         dllist_remove_node(&psFreeList->sNode);
1483         OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
1484
1485         SyncPrimFree(psFreeList->psCleanupSync);
1486
1487         /* free Freelist */
1488         OSFreeMem(psFreeList);
1489
1490         return eError;
1491 }
1492
1493
1494
1495 /*
1496         RGXAddBlockToFreeListKM
1497 */
1498
1499 IMG_EXPORT
1500 PVRSRV_ERROR RGXAddBlockToFreeListKM(RGX_FREELIST *psFreeList,
1501                                                                                 IMG_UINT32 ui32NumPages)
1502 {
1503         PVRSRV_ERROR eError;
1504
1505         /* Check if we have reference to freelist's PMR */
1506         if (psFreeList->psFreeListPMR == IMG_NULL)
1507         {
1508                 PVR_DPF((PVR_DBG_ERROR, "Freelist is not configured for grow"));
1509                 return PVRSRV_ERROR_INVALID_PARAMS;
1510         }
1511
1512         /* grow freelist */
1513         eError = RGXGrowFreeList(psFreeList,
1514                                                         ui32NumPages,
1515                                                         &psFreeList->sMemoryBlockHead);
1516         if(eError == PVRSRV_OK)
1517         {
1518                 /* update freelist data in firmware */
1519                 _UpdateFwFreelistSize(psFreeList, IMG_TRUE, ui32NumPages);
1520
1521                 psFreeList->ui32NumGrowReqByApp++;
1522
1523 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
1524                         /* Update Stats */
1525                         PVRSRVStatsUpdateFreelistStats(1, /* Add 1 to the appropriate counter (Requests by App)*/
1526                                        0,
1527                                        psFreeList->ui32InitFLPages,
1528                                        psFreeList->ui32NumHighPages,
1529                                        psFreeList->ownerPid);
1530
1531 #endif
1532         }
1533
1534         return eError;
1535 }
1536
1537 /*
1538         RGXRemoveBlockFromFreeListKM
1539 */
1540
1541 IMG_EXPORT
1542 PVRSRV_ERROR RGXRemoveBlockFromFreeListKM(RGX_FREELIST *psFreeList)
1543 {
1544         PVRSRV_ERROR eError;
1545
1546         /*
1547          * Make sure the pages part of the memory block are not in use anymore.
1548          * Instruct the firmware to update the freelist pointers accordingly.
1549          */
1550
1551         eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead,
1552                                                                 psFreeList);
1553
1554         return eError;
1555 }
1556
1557
1558 /*
1559         RGXCreateRenderTarget
1560 */
1561 IMG_EXPORT
1562 PVRSRV_ERROR RGXCreateRenderTarget(PVRSRV_DEVICE_NODE   *psDeviceNode,
1563                                                                    IMG_DEV_VIRTADDR             psVHeapTableDevVAddr,
1564                                                                    RGX_RT_CLEANUP_DATA  **ppsCleanupData,
1565                                                                    IMG_UINT32                   *sRenderTargetFWDevVAddr)
1566 {
1567         PVRSRV_ERROR                    eError = PVRSRV_OK;
1568         RGXFWIF_RENDER_TARGET   *psRenderTarget;
1569         RGXFWIF_DEV_VIRTADDR    pFirmwareAddr;
1570         PVRSRV_RGXDEV_INFO              *psDevInfo = psDeviceNode->pvDevice;
1571         RGX_RT_CLEANUP_DATA             *psCleanupData;
1572
1573         psCleanupData = OSAllocMem(sizeof(*psCleanupData));
1574         if (psCleanupData == IMG_NULL)
1575         {
1576                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1577                 goto err_out;
1578         }
1579
1580         OSMemSet(psCleanupData, 0, sizeof(*psCleanupData));
1581         psCleanupData->psDeviceNode = psDeviceNode;
1582         /*
1583          * This FW render target context is only mapped into kernel for initialisation.
1584          * Otherwise this allocation is only used by the FW.
1585          * Therefore the GPU cache doesn't need coherency,
1586          * and write-combine is suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
1587          */
1588         eError = DevmemFwAllocate(psDevInfo,
1589                                                         sizeof(*psRenderTarget),
1590                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1591                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1592                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1593                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1594                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1595                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1596                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1597                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1598                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1599                                                         "FirmwareRenderTarget",
1600                                                         &psCleanupData->psRenderTargetMemDesc);
1601         if (eError != PVRSRV_OK)
1602         {
1603                 PVR_DPF((PVR_DBG_ERROR, "RGXCreateRenderTarget: DevmemAllocate for Render Target failed"));
1604                 goto err_free;
1605         }
1606         RGXSetFirmwareAddress(&pFirmwareAddr, psCleanupData->psRenderTargetMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1607         *sRenderTargetFWDevVAddr = pFirmwareAddr.ui32Addr;
1608
1609         eError = DevmemAcquireCpuVirtAddr(psCleanupData->psRenderTargetMemDesc, (IMG_VOID **)&psRenderTarget);
1610         PVR_LOGG_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", err_fwalloc);
1611
1612         psRenderTarget->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
1613         psRenderTarget->bTACachesNeedZeroing = IMG_FALSE;
1614         PDUMPCOMMENT("Dump RenderTarget");
1615         DevmemPDumpLoadMem(psCleanupData->psRenderTargetMemDesc, 0, sizeof(*psRenderTarget), PDUMP_FLAGS_CONTINUOUS);
1616         DevmemReleaseCpuVirtAddr(psCleanupData->psRenderTargetMemDesc);
1617
1618         *ppsCleanupData = psCleanupData;
1619
1620 err_out:
1621         return eError;
1622
1623 err_free:
1624         OSFreeMem(psCleanupData);
1625         goto err_out;
1626
1627 err_fwalloc:
1628         DevmemFwFree(psCleanupData->psRenderTargetMemDesc);
1629         goto err_free;
1630
1631 }
1632
1633
1634 /*
1635         RGXDestroyRenderTarget
1636 */
1637 IMG_EXPORT
1638 PVRSRV_ERROR RGXDestroyRenderTarget(RGX_RT_CLEANUP_DATA *psCleanupData)
1639 {
1640         RGXUnsetFirmwareAddress(psCleanupData->psRenderTargetMemDesc);
1641
1642         /*
1643                 Note:
1644                 When we get RT cleanup in the FW call that instead
1645         */
1646         /* Flush the the SLC before freeing */
1647         {
1648                 RGXFWIF_KCCB_CMD sFlushInvalCmd;
1649                 PVRSRV_ERROR eError;
1650                 PVRSRV_DEVICE_NODE *psDeviceNode = psCleanupData->psDeviceNode;
1651
1652                 /* Schedule the SLC flush command ... */
1653         #if defined(PDUMP)
1654                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Submit SLC flush and invalidate");
1655         #endif
1656                 sFlushInvalCmd.eCmdType = RGXFWIF_KCCB_CMD_SLCFLUSHINVAL;
1657                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.bInval = IMG_TRUE;
1658                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.eDM = RGXFWIF_DM_2D; //Covers all of Sidekick
1659                 sFlushInvalCmd.uCmdData.sSLCFlushInvalData.psContext.ui32Addr = 0;
1660
1661                 eError = RGXSendCommandWithPowLock(psDeviceNode->pvDevice,
1662                                                                                         RGXFWIF_DM_GP,
1663                                                                                         &sFlushInvalCmd,
1664                                                                                         sizeof(sFlushInvalCmd),
1665                                                                                         IMG_TRUE);
1666                 if (eError != PVRSRV_OK)
1667                 {
1668                         PVR_DPF((PVR_DBG_ERROR,"RGXDestroyRenderTarget: Failed to schedule SLC flush command with error (%u)", eError));
1669                 }
1670                 else
1671                 {
1672                         /* Wait for the SLC flush to complete */
1673                         eError = RGXWaitForFWOp(psDeviceNode->pvDevice, RGXFWIF_DM_GP, psDeviceNode->psSyncPrim, IMG_TRUE);
1674                         if (eError != PVRSRV_OK)
1675                         {
1676                                 PVR_DPF((PVR_DBG_ERROR,"RGXDestroyRenderTarget: SLC flush and invalidate aborted with error (%u)", eError));
1677                         }
1678                 }
1679         }
1680
1681         DevmemFwFree(psCleanupData->psRenderTargetMemDesc);
1682         OSFreeMem(psCleanupData);
1683         return PVRSRV_OK;
1684 }
1685
1686 /*
1687         RGXCreateZSBuffer
1688 */
1689 IMG_EXPORT
1690 PVRSRV_ERROR RGXCreateZSBufferKM(PVRSRV_DEVICE_NODE     *psDeviceNode,
1691                                                                 DEVMEMINT_RESERVATION   *psReservation,
1692                                                                 PMR                                     *psPMR,
1693                                                                 PVRSRV_MEMALLOCFLAGS_T  uiMapFlags,
1694                                                                 RGX_ZSBUFFER_DATA **ppsZSBuffer,
1695                                                                 IMG_UINT32 *pui32ZSBufferFWDevVAddr)
1696 {
1697         PVRSRV_ERROR                            eError;
1698         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
1699         RGXFWIF_FWZSBUFFER                      *psFWZSBuffer;
1700         RGX_ZSBUFFER_DATA                       *psZSBuffer;
1701         DEVMEM_MEMDESC                          *psFWZSBufferMemDesc;
1702         IMG_BOOL                                        bOnDemand = ((uiMapFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC) > 0);
1703
1704         /* Allocate host data structure */
1705         psZSBuffer = OSAllocMem(sizeof(*psZSBuffer));
1706         if (psZSBuffer == IMG_NULL)
1707         {
1708                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate cleanup data structure for ZS-Buffer"));
1709                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1710                 goto ErrorAllocCleanup;
1711         }
1712         OSMemSet(psZSBuffer, 0, sizeof(*psZSBuffer));
1713
1714         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
1715                                                    &psZSBuffer->psCleanupSync,
1716                                                    "ta3d zs buffer cleanup");
1717         if (eError != PVRSRV_OK)
1718         {
1719                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate cleanup sync (0x%x)",
1720                                 eError));
1721                 goto ErrorSyncAlloc;
1722         }
1723
1724         /* Populate Host data */
1725         psZSBuffer->psDevInfo = psDevInfo;
1726         psZSBuffer->psReservation = psReservation;
1727         psZSBuffer->psPMR = psPMR;
1728         psZSBuffer->uiMapFlags = uiMapFlags;
1729         psZSBuffer->ui32RefCount = 0;
1730         psZSBuffer->bOnDemand = bOnDemand;
1731     if (bOnDemand)
1732     {
1733         psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
1734         psZSBuffer->psMapping = IMG_NULL;
1735
1736                 OSLockAcquire(psDevInfo->hLockZSBuffer);
1737         dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
1738                 OSLockRelease(psDevInfo->hLockZSBuffer);
1739     }
1740
1741         /* Allocate firmware memory for ZS-Buffer. */
1742         PDUMPCOMMENT("Allocate firmware ZS-Buffer data structure");
1743         eError = DevmemFwAllocate(psDevInfo,
1744                                                         sizeof(*psFWZSBuffer),
1745                                                         PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
1746                                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
1747                                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE |
1748                                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
1749                                                         PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
1750                                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE |
1751                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
1752                                                         PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE |
1753                                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE,
1754                                                         "FirmwareZSBuffer",
1755                                                         &psFWZSBufferMemDesc);
1756         if (eError != PVRSRV_OK)
1757         {
1758                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to allocate firmware ZS-Buffer (%u)", eError));
1759                 goto ErrorAllocFWZSBuffer;
1760         }
1761         psZSBuffer->psZSBufferMemDesc = psFWZSBufferMemDesc;
1762
1763         /* Temporarily map the firmware render context to the kernel. */
1764         eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
1765                                       (IMG_VOID **)&psFWZSBuffer);
1766         if (eError != PVRSRV_OK)
1767         {
1768                 PVR_DPF((PVR_DBG_ERROR,"RGXCreateZSBufferKM: Failed to map firmware ZS-Buffer (%u)", eError));
1769                 goto ErrorAcquireFWZSBuffer;
1770         }
1771
1772         /* Populate FW ZS-Buffer data structure */
1773         psFWZSBuffer->bOnDemand = bOnDemand;
1774         psFWZSBuffer->eState = (bOnDemand) ? RGXFWIF_ZSBUFFER_UNBACKED : RGXFWIF_ZSBUFFER_BACKED;
1775         psFWZSBuffer->ui32ZSBufferID = psZSBuffer->ui32ZSBufferID;
1776
1777         /* Get firmware address of ZS-Buffer. */
1778         RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
1779
1780         /* Dump the ZS-Buffer and the memory content */
1781         PDUMPCOMMENT("Dump firmware ZS-Buffer");
1782         DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
1783
1784         /* Release address acquired above. */
1785         DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
1786
1787
1788         /* define return value */
1789         *ppsZSBuffer = psZSBuffer;
1790         *pui32ZSBufferFWDevVAddr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
1791
1792         PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
1793                                                         psZSBuffer,
1794                                                         (bOnDemand) ? "On-Demand": "Up-front"));
1795
1796         psZSBuffer->owner=OSGetCurrentProcessIDKM();
1797
1798         return PVRSRV_OK;
1799
1800         /* error handling */
1801
1802 ErrorAcquireFWZSBuffer:
1803         DevmemFwFree(psFWZSBufferMemDesc);
1804
1805 ErrorAllocFWZSBuffer:
1806         SyncPrimFree(psZSBuffer->psCleanupSync);
1807
1808 ErrorSyncAlloc:
1809         OSFreeMem(psZSBuffer);
1810
1811 ErrorAllocCleanup:
1812         PVR_ASSERT(eError != PVRSRV_OK);
1813         return eError;
1814 }
1815
1816
1817 /*
1818         RGXDestroyZSBuffer
1819 */
1820 IMG_EXPORT
1821 PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
1822 {
1823         POS_LOCK hLockZSBuffer;
1824         PVRSRV_ERROR eError;
1825
1826         PVR_ASSERT(psZSBuffer);
1827         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
1828
1829         /* Request ZS Buffer cleanup */
1830         eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
1831                                                                                 psZSBuffer->sZSBufferFWDevVAddr,
1832                                                                                 psZSBuffer->psCleanupSync);
1833         if (eError != PVRSRV_ERROR_RETRY)
1834         {
1835                 /* Free the firmware render context. */
1836         RGXUnsetFirmwareAddress(psZSBuffer->psZSBufferMemDesc);
1837                 DevmemFwFree(psZSBuffer->psZSBufferMemDesc);
1838
1839             /* Remove Deferred Allocation from list */
1840                 if (psZSBuffer->bOnDemand)
1841                 {
1842                         OSLockAcquire(hLockZSBuffer);
1843                         PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
1844                         dllist_remove_node(&psZSBuffer->sNode);
1845                         OSLockRelease(hLockZSBuffer);
1846                 }
1847
1848                 SyncPrimFree(psZSBuffer->psCleanupSync);
1849
1850                 PVR_ASSERT(psZSBuffer->ui32RefCount == 0);
1851
1852                 PVR_DPF((PVR_DBG_MESSAGE,"ZS-Buffer [%p] destroyed",psZSBuffer));
1853
1854                 /* Free ZS-Buffer host data structure */
1855                 OSFreeMem(psZSBuffer);
1856
1857         }
1858
1859         return eError;
1860 }
1861
1862 PVRSRV_ERROR
1863 RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
1864 {
1865         POS_LOCK hLockZSBuffer;
1866         PVRSRV_ERROR eError;
1867
1868         if (!psZSBuffer)
1869         {
1870                 return PVRSRV_ERROR_INVALID_PARAMS;
1871         }
1872
1873         if ((psZSBuffer->uiMapFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC) == 0)
1874         {
1875                 /* Only deferred allocations can be populated */
1876                 return PVRSRV_ERROR_INVALID_PARAMS;
1877         }
1878
1879         PVR_DPF((PVR_DBG_MESSAGE,"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
1880                                                                 psZSBuffer,
1881                                                                 psZSBuffer->ui32ZSBufferID));
1882         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
1883
1884         OSLockAcquire(hLockZSBuffer);
1885
1886         if (psZSBuffer->ui32RefCount == 0)
1887         {
1888                 if (psZSBuffer->bOnDemand)
1889                 {
1890                         IMG_HANDLE hDevmemHeap;
1891
1892                         PVR_ASSERT(psZSBuffer->psMapping == IMG_NULL);
1893
1894                         /* Get Heap */
1895                         eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
1896                         PVR_ASSERT(psZSBuffer->psMapping == IMG_NULL);
1897
1898                         eError = DevmemIntMapPMR(hDevmemHeap,
1899                                                                         psZSBuffer->psReservation,
1900                                                                         psZSBuffer->psPMR,
1901                                                                         psZSBuffer->uiMapFlags,
1902                                                                         &psZSBuffer->psMapping);
1903                         if (eError != PVRSRV_OK)
1904                         {
1905                                 PVR_DPF((PVR_DBG_ERROR,"Unable populate ZS Buffer [%p, ID=0x%08x] with error %u",
1906                                                                                 psZSBuffer,
1907                                                                                 psZSBuffer->ui32ZSBufferID,
1908                                                                                 eError));
1909                                 OSLockRelease(hLockZSBuffer);
1910                                 return eError;
1911
1912                         }
1913                         PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
1914                                                                                 psZSBuffer,
1915                                                                                 psZSBuffer->ui32ZSBufferID));
1916                 }
1917         }
1918
1919         /* Increase refcount*/
1920         psZSBuffer->ui32RefCount++;
1921
1922         OSLockRelease(hLockZSBuffer);
1923
1924         return PVRSRV_OK;
1925 }
1926
1927
1928 PVRSRV_ERROR
1929 RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
1930                                         RGX_POPULATION **ppsPopulation)
1931 {
1932         RGX_POPULATION *psPopulation;
1933         PVRSRV_ERROR eError;
1934
1935         psZSBuffer->ui32NumReqByApp++;
1936
1937 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
1938         PVRSRVStatsUpdateZSBufferStats(1,0,psZSBuffer->owner);
1939 #endif
1940
1941         /* Do the backing */
1942         eError = RGXBackingZSBuffer(psZSBuffer);
1943         if (eError != PVRSRV_OK)
1944         {
1945                 goto OnErrorBacking;
1946         }
1947
1948         /* Create the handle to the backing */
1949         psPopulation = OSAllocMem(sizeof(*psPopulation));
1950         if (psPopulation == IMG_NULL)
1951         {
1952                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1953                 goto OnErrorAlloc;
1954         }
1955
1956         psPopulation->psZSBuffer = psZSBuffer;
1957
1958         /* return value */
1959         *ppsPopulation = psPopulation;
1960
1961         return PVRSRV_OK;
1962
1963 OnErrorAlloc:
1964         RGXUnbackingZSBuffer(psZSBuffer);
1965
1966 OnErrorBacking:
1967         PVR_ASSERT(eError != PVRSRV_OK);
1968         return eError;
1969 }
1970
1971 PVRSRV_ERROR
1972 RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
1973 {
1974         POS_LOCK hLockZSBuffer;
1975         PVRSRV_ERROR eError;
1976
1977         if (!psZSBuffer)
1978         {
1979                 return PVRSRV_ERROR_INVALID_PARAMS;
1980         }
1981
1982         PVR_ASSERT(psZSBuffer->ui32RefCount);
1983
1984         PVR_DPF((PVR_DBG_MESSAGE,"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
1985                                                                 psZSBuffer,
1986                                                                 psZSBuffer->ui32ZSBufferID));
1987
1988         hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
1989
1990         OSLockAcquire(hLockZSBuffer);
1991
1992         if (psZSBuffer->bOnDemand)
1993         {
1994                 if (psZSBuffer->ui32RefCount == 1)
1995                 {
1996                         PVR_ASSERT(psZSBuffer->psMapping);
1997
1998                         eError = DevmemIntUnmapPMR(psZSBuffer->psMapping);
1999                         if (eError != PVRSRV_OK)
2000                         {
2001                                 PVR_DPF((PVR_DBG_ERROR,"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] with error %u",
2002                                                                                 psZSBuffer,
2003                                                                                 psZSBuffer->ui32ZSBufferID,
2004                                                                                 eError));
2005                                 OSLockRelease(hLockZSBuffer);
2006                                 return eError;
2007                         }
2008
2009                         PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
2010                                                                                 psZSBuffer,
2011                                                                                 psZSBuffer->ui32ZSBufferID));
2012                 }
2013         }
2014
2015         /* Decrease refcount*/
2016         psZSBuffer->ui32RefCount--;
2017
2018         OSLockRelease(hLockZSBuffer);
2019
2020         return PVRSRV_OK;
2021 }
2022
2023 PVRSRV_ERROR
2024 RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
2025 {
2026         PVRSRV_ERROR eError;
2027
2028         if (!psPopulation)
2029         {
2030                 return PVRSRV_ERROR_INVALID_PARAMS;
2031         }
2032
2033         eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
2034         if (eError != PVRSRV_OK)
2035         {
2036                 return eError;
2037         }
2038
2039         OSFreeMem(psPopulation);
2040
2041         return PVRSRV_OK;
2042 }
2043
2044 static IMG_BOOL _FindZSBuffer(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
2045 {
2046         DEVMEM_REF_LOOKUP *psRefLookUp = (DEVMEM_REF_LOOKUP *)pvCallbackData;
2047         RGX_ZSBUFFER_DATA *psZSBuffer;
2048
2049         psZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
2050
2051         if (psZSBuffer->ui32ZSBufferID == psRefLookUp->ui32ZSBufferID)
2052         {
2053                 psRefLookUp->psZSBuffer = psZSBuffer;
2054                 return IMG_FALSE;
2055         }
2056         else
2057         {
2058                 return IMG_TRUE;
2059         }
2060 }
2061
2062 IMG_VOID RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
2063                                                                                         IMG_UINT32 ui32ZSBufferID)
2064 {
2065         DEVMEM_REF_LOOKUP sLookUp;
2066         RGXFWIF_KCCB_CMD sTACCBCmd;
2067         PVRSRV_ERROR eError;
2068
2069         PVR_ASSERT(psDevInfo);
2070
2071         /* scan all deferred allocations */
2072         sLookUp.ui32ZSBufferID = ui32ZSBufferID;
2073         sLookUp.psZSBuffer = IMG_NULL;
2074
2075         OSLockAcquire(psDevInfo->hLockZSBuffer);
2076         dllist_foreach_node(&psDevInfo->sZSBufferHead, _FindZSBuffer, (IMG_PVOID)&sLookUp);
2077         OSLockRelease(psDevInfo->hLockZSBuffer);
2078
2079         if (sLookUp.psZSBuffer)
2080         {
2081                 IMG_BOOL bBackingDone = IMG_TRUE;
2082
2083                 /* Populate ZLS */
2084                 eError = RGXBackingZSBuffer(sLookUp.psZSBuffer);
2085                 if (eError != PVRSRV_OK)
2086                 {
2087                         PVR_DPF((PVR_DBG_ERROR,"Populating ZS-Buffer failed failed with error %u (ID = 0x%08x)", eError, ui32ZSBufferID));
2088                         bBackingDone = IMG_FALSE;
2089                 }
2090
2091                 /* send confirmation */
2092                 sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
2093                 sTACCBCmd.uCmdData.sZSBufferBackingData.psZSBufferFWDevVAddr = sLookUp.psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
2094                 sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
2095
2096                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
2097                 {
2098                         eError = RGXScheduleCommand(psDevInfo,
2099                                                                                                 RGXFWIF_DM_TA,
2100                                                                                                 &sTACCBCmd,
2101                                                                                                 sizeof(sTACCBCmd),
2102                                                                                                 IMG_FALSE);
2103                         if (eError != PVRSRV_ERROR_RETRY)
2104                         {
2105                                 break;
2106                         }
2107                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
2108                 } END_LOOP_UNTIL_TIMEOUT();
2109
2110                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
2111                 PVR_ASSERT(eError == PVRSRV_OK);
2112
2113                 sLookUp.psZSBuffer->ui32NumReqByFW++;
2114
2115 #if defined(PVRSRV_ENABLE_PROCESS_STATS)
2116                 PVRSRVStatsUpdateZSBufferStats(0,1,sLookUp.psZSBuffer->owner);
2117 #endif
2118
2119         }
2120         else
2121         {
2122                 PVR_DPF((PVR_DBG_ERROR,"ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)", sLookUp.ui32ZSBufferID));
2123         }
2124 }
2125
2126 IMG_VOID RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
2127                                                                                         IMG_UINT32 ui32ZSBufferID)
2128 {
2129         DEVMEM_REF_LOOKUP sLookUp;
2130         RGXFWIF_KCCB_CMD sTACCBCmd;
2131         PVRSRV_ERROR eError;
2132
2133         PVR_ASSERT(psDevInfo);
2134
2135         /* scan all deferred allocations */
2136         sLookUp.ui32ZSBufferID = ui32ZSBufferID;
2137         sLookUp.psZSBuffer = IMG_NULL;
2138
2139         OSLockAcquire(psDevInfo->hLockZSBuffer);
2140         dllist_foreach_node(&psDevInfo->sZSBufferHead, _FindZSBuffer, (IMG_PVOID)&sLookUp);
2141         OSLockRelease(psDevInfo->hLockZSBuffer);
2142
2143         if (sLookUp.psZSBuffer)
2144         {
2145                 /* Unpopulate ZLS */
2146                 eError = RGXUnbackingZSBuffer(sLookUp.psZSBuffer);
2147                 if (eError != PVRSRV_OK)
2148                 {
2149                         PVR_DPF((PVR_DBG_ERROR,"UnPopulating ZS-Buffer failed failed with error %u (ID = 0x%08x)", eError, ui32ZSBufferID));
2150                         PVR_ASSERT(IMG_FALSE);
2151                 }
2152
2153                 /* send confirmation */
2154                 sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
2155                 sTACCBCmd.uCmdData.sZSBufferBackingData.psZSBufferFWDevVAddr = sLookUp.psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
2156                 sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
2157
2158                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
2159                 {
2160                         eError = RGXScheduleCommand(psDevInfo,
2161                                                                                                 RGXFWIF_DM_TA,
2162                                                                                                 &sTACCBCmd,
2163                                                                                                 sizeof(sTACCBCmd),
2164                                                                                                 IMG_FALSE);
2165                         if (eError != PVRSRV_ERROR_RETRY)
2166                         {
2167                                 break;
2168                         }
2169                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
2170                 } END_LOOP_UNTIL_TIMEOUT();
2171
2172                 /* Kernel CCB should never fill up, as the FW is processing them right away  */
2173                 PVR_ASSERT(eError == PVRSRV_OK);
2174
2175         }
2176         else
2177         {
2178                 PVR_DPF((PVR_DBG_ERROR,"ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)", sLookUp.ui32ZSBufferID));
2179         }
2180 }
2181
2182 static
2183 PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
2184                                                           PVRSRV_DEVICE_NODE *psDeviceNode,
2185                                                           DEVMEM_MEMDESC *psAllocatedMemDesc,
2186                                                           IMG_UINT32 ui32AllocatedOffset,
2187                                                           DEVMEM_MEMDESC *psFWMemContextMemDesc,
2188                                                           IMG_DEV_VIRTADDR sVDMCallStackAddr,
2189                                                           IMG_UINT32 ui32Priority,
2190                                                           RGX_COMMON_CONTEXT_INFO *psInfo,
2191                                                           RGX_SERVER_RC_TA_DATA *psTAData)
2192 {
2193         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
2194         RGXFWIF_TACTX_STATE *psContextState;
2195         PVRSRV_ERROR eError;
2196         /*
2197                 Allocate device memory for the firmware GPU context suspend state.
2198                 Note: the FW reads/writes the state to memory by accessing the GPU register interface.
2199         */
2200         PDUMPCOMMENT("Allocate RGX firmware TA context suspend state");
2201
2202         eError = DevmemFwAllocate(psDevInfo,
2203                                                           sizeof(RGXFWIF_TACTX_STATE),
2204                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2205                                                           "FirmwareTAContextState",
2206                                                           &psTAData->psContextStateMemDesc);
2207         if (eError != PVRSRV_OK)
2208         {
2209                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU context suspend state (%u)",
2210                                 eError));
2211                 goto fail_tacontextsuspendalloc;
2212         }
2213
2214         eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
2215                                       (IMG_VOID **)&psContextState);
2216         if (eError != PVRSRV_OK)
2217         {
2218                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to map firmware render context state (%u)",
2219                                 eError));
2220                 goto fail_suspendcpuvirtacquire;
2221         }
2222         psContextState->uTAReg_VDM_CALL_STACK_POINTER_Init = sVDMCallStackAddr.uiAddr;
2223         DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
2224
2225         eError = FWCommonContextAllocate(psConnection,
2226                                                                          psDeviceNode,
2227                                                                          "TA",
2228                                                                          psAllocatedMemDesc,
2229                                                                          ui32AllocatedOffset,
2230                                                                          psFWMemContextMemDesc,
2231                                                                          psTAData->psContextStateMemDesc,
2232                                                                          RGX_CCB_SIZE_LOG2,
2233                                                                          ui32Priority,
2234                                                                          psInfo,
2235                                                                          &psTAData->psServerCommonContext);
2236         if (eError != PVRSRV_OK)
2237         {
2238                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to init TA fw common context (%u)",
2239                                 eError));
2240                 goto fail_tacommoncontext;
2241         }
2242
2243         /*
2244          * Dump the FW 3D context suspend state buffer
2245          */
2246         PDUMPCOMMENT("Dump the TA context suspend state buffer");
2247         DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
2248                                            0,
2249                                            sizeof(RGXFWIF_TACTX_STATE),
2250                                            PDUMP_FLAGS_CONTINUOUS);
2251
2252         psTAData->ui32Priority = ui32Priority;
2253         return PVRSRV_OK;
2254
2255 fail_tacommoncontext:
2256 fail_suspendcpuvirtacquire:
2257         DevmemFwFree(psTAData->psContextStateMemDesc);
2258 fail_tacontextsuspendalloc:
2259         PVR_ASSERT(eError != PVRSRV_OK);
2260
2261         return eError;
2262 }
2263
2264 static
2265 PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
2266                                                           PVRSRV_DEVICE_NODE *psDeviceNode,
2267                                                           DEVMEM_MEMDESC *psAllocatedMemDesc,
2268                                                           IMG_UINT32 ui32AllocatedOffset,
2269                                                           DEVMEM_MEMDESC *psFWMemContextMemDesc,
2270                                                           IMG_UINT32 ui32Priority,
2271                                                           RGX_COMMON_CONTEXT_INFO *psInfo,
2272                                                           RGX_SERVER_RC_3D_DATA *ps3DData)
2273 {
2274         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
2275         PVRSRV_ERROR eError;
2276
2277         /*
2278                 Allocate device memory for the firmware GPU context suspend state.
2279                 Note: the FW reads/writes the state to memory by accessing the GPU register interface.
2280         */
2281         PDUMPCOMMENT("Allocate RGX firmware 3D context suspend state");
2282
2283         eError = DevmemFwAllocate(psDevInfo,
2284                                                           sizeof(RGXFWIF_3DCTX_STATE),
2285                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2286                                                           "Firmware3DContextState",
2287                                                           &ps3DData->psContextStateMemDesc);
2288         if (eError != PVRSRV_OK)
2289         {
2290                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU context suspend state (%u)",
2291                                 eError));
2292                 goto fail_3dcontextsuspendalloc;
2293         }
2294
2295         eError = FWCommonContextAllocate(psConnection,
2296                                                                          psDeviceNode,
2297                                                                          "3D",
2298                                                                          psAllocatedMemDesc,
2299                                                                          ui32AllocatedOffset,
2300                                                                          psFWMemContextMemDesc,
2301                                                                          ps3DData->psContextStateMemDesc,
2302                                                                          RGX_CCB_SIZE_LOG2,
2303                                                                          ui32Priority,
2304                                                                          psInfo,
2305                                                                          &ps3DData->psServerCommonContext);
2306         if (eError != PVRSRV_OK)
2307         {
2308                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to init 3D fw common context (%u)",
2309                                 eError));
2310                 goto fail_3dcommoncontext;
2311         }
2312
2313         /*
2314          * Dump the FW 3D context suspend state buffer
2315          */
2316         PDUMPCOMMENT("Dump the 3D context suspend state buffer");
2317         DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
2318                                            0,
2319                                            sizeof(RGXFWIF_3DCTX_STATE),
2320                                            PDUMP_FLAGS_CONTINUOUS);
2321
2322         ps3DData->ui32Priority = ui32Priority;
2323         return PVRSRV_OK;
2324
2325 fail_3dcommoncontext:
2326         DevmemFwFree(ps3DData->psContextStateMemDesc);
2327 fail_3dcontextsuspendalloc:
2328         PVR_ASSERT(eError != PVRSRV_OK);
2329
2330         return eError;
2331 }
2332
2333
2334 /*
2335  * PVRSRVRGXCreateRenderContextKM
2336  */
2337 IMG_EXPORT
2338 PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA                             *psConnection,
2339                                                                                         PVRSRV_DEVICE_NODE                      *psDeviceNode,
2340                                                                                         IMG_UINT32                                      ui32Priority,
2341                                                                                         IMG_DEV_VIRTADDR                        sMCUFenceAddr,
2342                                                                                         IMG_DEV_VIRTADDR                        sVDMCallStackAddr,
2343                                                                                         IMG_UINT32                                      ui32FrameworkRegisterSize,
2344                                                                                         IMG_PBYTE                                       pabyFrameworkRegisters,
2345                                                                                         IMG_HANDLE                                      hMemCtxPrivData,
2346                                                                                         RGX_SERVER_RENDER_CONTEXT       **ppsRenderContext)
2347 {
2348         PVRSRV_ERROR                            eError;
2349         PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
2350         RGX_SERVER_RENDER_CONTEXT       *psRenderContext;
2351         DEVMEM_MEMDESC                          *psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
2352         RGX_COMMON_CONTEXT_INFO         sInfo;
2353
2354         /* Prepare cleanup structure */
2355         *ppsRenderContext = IMG_NULL;
2356         psRenderContext = OSAllocMem(sizeof(*psRenderContext));
2357         if (psRenderContext == IMG_NULL)
2358         {
2359                 return PVRSRV_ERROR_OUT_OF_MEMORY;
2360         }
2361
2362         OSMemSet(psRenderContext, 0, sizeof(*psRenderContext));
2363         psRenderContext->psDeviceNode = psDeviceNode;
2364
2365         /*
2366                 Create the FW render context, this has the TA and 3D FW common
2367                 contexts embedded within it
2368         */
2369         eError = DevmemFwAllocate(psDevInfo,
2370                                                           sizeof(RGXFWIF_FWRENDERCONTEXT),
2371                                                           RGX_FWCOMCTX_ALLOCFLAGS,
2372                                                           "FirmwareRenderContext",
2373                                                           &psRenderContext->psFWRenderContextMemDesc);
2374         if (eError != PVRSRV_OK)
2375         {
2376                 goto fail_fwrendercontext;
2377         }
2378
2379         /*
2380                 As the common context alloc will dump the TA and 3D common contexts
2381                 after the've been setup we skip of the 2 common contexts and dump the
2382                 rest of the structure
2383         */
2384         PDUMPCOMMENT("Dump shared part of render context context");
2385         DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc,
2386                                            (sizeof(RGXFWIF_FWCOMMONCONTEXT) * 2),
2387                                            sizeof(RGXFWIF_FWRENDERCONTEXT) - (sizeof(RGXFWIF_FWCOMMONCONTEXT) * 2),
2388                                            PDUMP_FLAGS_CONTINUOUS);
2389
2390         /* Allocate cleanup sync */
2391         eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
2392                                                    &psRenderContext->psCleanupSync,
2393                                                    "ta3d render context cleanup");
2394         if (eError != PVRSRV_OK)
2395         {
2396                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate cleanup sync (0x%x)",
2397                                 eError));
2398                 goto fail_syncalloc;
2399         }
2400
2401         /*
2402          * Create the FW framework buffer
2403          */
2404         eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
2405                                                                                 &psRenderContext->psFWFrameworkMemDesc,
2406                                                                                 ui32FrameworkRegisterSize);
2407         if (eError != PVRSRV_OK)
2408         {
2409                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to allocate firmware GPU framework state (%u)",
2410                                 eError));
2411                 goto fail_frameworkcreate;
2412         }
2413
2414         /* Copy the Framework client data into the framework buffer */
2415         eError = PVRSRVRGXFrameworkCopyCommand(psRenderContext->psFWFrameworkMemDesc,
2416                                                                                    pabyFrameworkRegisters,
2417                                                                                    ui32FrameworkRegisterSize);
2418         if (eError != PVRSRV_OK)
2419         {
2420                 PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXCreateRenderContextKM: Failed to populate the framework buffer (%u)",
2421                                 eError));
2422                 goto fail_frameworkcopy;
2423         }
2424
2425         sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
2426         sInfo.psMCUFenceAddr = &sMCUFenceAddr;
2427
2428         eError = _CreateTAContext(psConnection,
2429                                                           psDeviceNode,
2430                                                           psRenderContext->psFWRenderContextMemDesc,
2431                                                           offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
2432                                                           psFWMemContextMemDesc,
2433                                                           sVDMCallStackAddr,
2434                                                           ui32Priority,
2435                                                           &sInfo,
2436                                                           &psRenderContext->sTAData);
2437         if (eError != PVRSRV_OK)
2438         {
2439                 goto fail_tacontext;
2440         }
2441
2442         eError = _Create3DContext(psConnection,
2443                                                           psDeviceNode,
2444                                                           psRenderContext->psFWRenderContextMemDesc,
2445                                                           offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
2446                                                           psFWMemContextMemDesc,
2447                                                           ui32Priority,
2448                                                           &sInfo,
2449                                                           &psRenderContext->s3DData);
2450         if (eError != PVRSRV_OK)
2451         {
2452                 goto fail_3dcontext;
2453         }
2454
2455         SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
2456         SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
2457         SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
2458         SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
2459
2460         {
2461                 PVRSRV_RGXDEV_INFO                      *psDevInfo = psDeviceNode->pvDevice;
2462
2463                 OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock, DEVINFO_RENDERLIST);
2464                 dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
2465                 OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2466         }
2467
2468         *ppsRenderContext= psRenderContext;
2469         return PVRSRV_OK;
2470
2471 fail_3dcontext:
2472         _DestroyTAContext(&psRenderContext->sTAData,
2473                                           psDeviceNode,
2474                                           psRenderContext->psCleanupSync);
2475 fail_tacontext:
2476 fail_frameworkcopy:
2477         DevmemFwFree(psRenderContext->psFWFrameworkMemDesc);
2478 fail_frameworkcreate:
2479         SyncPrimFree(psRenderContext->psCleanupSync);
2480 fail_syncalloc:
2481         DevmemFwFree(psRenderContext->psFWRenderContextMemDesc);
2482 fail_fwrendercontext:
2483         OSFreeMem(psRenderContext);
2484         PVR_ASSERT(eError != PVRSRV_OK);
2485
2486         return eError;
2487 }
2488
2489 /*
2490  * PVRSRVRGXDestroyRenderContextKM
2491  */
2492 IMG_EXPORT
2493 PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
2494 {
2495         PVRSRV_ERROR                            eError;
2496         PVRSRV_RGXDEV_INFO      *psDevInfo = psRenderContext->psDeviceNode->pvDevice;
2497
2498         /* remove node from list before calling destroy - as destroy, if successful
2499          * will invalidate the node
2500          * must be re-added if destroy fails
2501          */
2502         OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock, DEVINFO_RENDERLIST);
2503         dllist_remove_node(&(psRenderContext->sListNode));
2504         OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2505
2506         /* Cleanup the TA if we haven't already */
2507         if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
2508         {
2509                 eError = _DestroyTAContext(&psRenderContext->sTAData,
2510                                                                    psRenderContext->psDeviceNode,
2511                                                                    psRenderContext->psCleanupSync);
2512                 if (eError != PVRSRV_ERROR_RETRY)
2513                 {
2514                         psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
2515                 }
2516                 else
2517                 {
2518                         goto e0;
2519                 }
2520         }
2521
2522         /* Cleanup the 3D if we haven't already */
2523         if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
2524         {
2525                 eError = _Destroy3DContext(&psRenderContext->s3DData,
2526                                                                    psRenderContext->psDeviceNode,
2527                                                                    psRenderContext->psCleanupSync);
2528                 if (eError != PVRSRV_ERROR_RETRY)
2529                 {
2530                         psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
2531                 }
2532                 else
2533                 {
2534                         goto e0;
2535                 }
2536         }
2537
2538         /*
2539                 Only if both TA and 3D contexts have been cleaned up can we
2540                 free the shared resources
2541         */
2542         if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
2543         {
2544                 RGXFWIF_FWRENDERCONTEXT *psFWRenderContext;
2545
2546                 /* Update SPM statistics */
2547                 eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
2548                                               (IMG_VOID **)&psFWRenderContext);
2549                 if (eError == PVRSRV_OK)
2550                 {
2551                         DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
2552                 }
2553                 else
2554                 {
2555                         PVR_DPF((PVR_DBG_ERROR,"PVRSRVRGXDestroyRenderContextKM: Failed to map firmware render context (%u)",
2556                                         eError));
2557                 }
2558
2559                 /* Free the framework buffer */
2560                 DevmemFwFree(psRenderContext->psFWFrameworkMemDesc);
2561
2562                 /* Free the firmware render context */
2563                 DevmemFwFree(psRenderContext->psFWRenderContextMemDesc);
2564
2565                 /* Free the cleanup sync */
2566                 SyncPrimFree(psRenderContext->psCleanupSync);
2567
2568                 SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
2569                 SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
2570                 SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
2571                 SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
2572
2573                 OSFreeMem(psRenderContext);
2574         }
2575
2576         return PVRSRV_OK;
2577
2578 e0:
2579         OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock, DEVINFO_RENDERLIST);
2580         dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
2581         OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
2582         return eError;
2583 }
2584
2585
2586
2587 /* 1 command for the TA */
2588 static RGX_CCB_CMD_HELPER_DATA sTACmdHelperData;
2589 /* Up to 3 commands for the 3D (partial render fence, partial reader, and render) */
2590 static RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[3];
2591
2592 /*
2593  * PVRSRVRGXKickTA3DKM
2594  */
2595 IMG_EXPORT
2596 PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT      *psRenderContext,
2597                                                                  IMG_UINT32                                     ui32ClientTAFenceCount,
2598                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClientTAFenceSyncPrimBlock,
2599                                                                  IMG_UINT32                                     *paui32ClientTAFenceSyncOffset,
2600                                                                  IMG_UINT32                                     *paui32ClientTAFenceValue,
2601                                                                  IMG_UINT32                                     ui32ClientTAUpdateCount,
2602                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClientTAUpdateSyncPrimBlock,
2603                                                                  IMG_UINT32                                     *paui32ClientTAUpdateSyncOffset,
2604                                                                  IMG_UINT32                                     *paui32ClientTAUpdateValue,
2605                                                                  IMG_UINT32                                     ui32ServerTASyncPrims,
2606                                                                  IMG_UINT32                                     *paui32ServerTASyncFlags,
2607                                                                  SERVER_SYNC_PRIMITIVE          **pasServerTASyncs,
2608                                                                  IMG_UINT32                                     ui32Client3DFenceCount,
2609                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClient3DFenceSyncPrimBlock,
2610                                                                  IMG_UINT32                                     *paui32Client3DFenceSyncOffset,
2611                                                                  IMG_UINT32                                     *paui32Client3DFenceValue,
2612                                                                  IMG_UINT32                                     ui32Client3DUpdateCount,
2613                                                                  SYNC_PRIMITIVE_BLOCK                           **apsClient3DUpdateSyncPrimBlock,
2614                                                                  IMG_UINT32                                     *paui32Client3DUpdateSyncOffset,
2615                                                                  IMG_UINT32                                     *paui32Client3DUpdateValue,
2616                                                                  IMG_UINT32                                     ui32Server3DSyncPrims,
2617                                                                  IMG_UINT32                                     *paui32Server3DSyncFlags,
2618                                                                  SERVER_SYNC_PRIMITIVE          **pasServer3DSyncs,
2619                                                                  SYNC_PRIMITIVE_BLOCK                           *psPRFenceSyncPrimBlock,
2620                                                                  IMG_UINT32                                     ui32PRFenceSyncOffset,
2621                                                                  IMG_UINT32                                     ui32PRFenceValue,
2622                                                                  IMG_UINT32                                     ui32NumFenceFDs,
2623                                                                  IMG_INT32                                      *ai32FenceFDs,
2624                                                                  IMG_UINT32                                     ui32TACmdSize,
2625                                                                  IMG_PBYTE                                      pui8TADMCmd,
2626                                                                  IMG_UINT32                                     ui323DPRCmdSize,
2627                                                                  IMG_PBYTE                                      pui83DPRDMCmd,
2628                                                                  IMG_UINT32                                     ui323DCmdSize,
2629                                                                  IMG_PBYTE                                      pui83DDMCmd,
2630                                                                  IMG_UINT32                                     ui32ExtJobRef,
2631                                                                  IMG_UINT32                                     ui32IntJobRef,
2632                                                                  IMG_BOOL                                       bLastTAInScene,
2633                                                                  IMG_BOOL                                       bKickTA,
2634                                                                  IMG_BOOL                                       bKickPR,
2635                                                                  IMG_BOOL                                       bKick3D,
2636                                                                  IMG_BOOL                                       bAbort,
2637                                                                  IMG_BOOL                                       bPDumpContinuous,
2638                                                                  RGX_RTDATA_CLEANUP_DATA        *psRTDataCleanup,
2639                                                                  RGX_ZSBUFFER_DATA              *psZBuffer,
2640                                                                  RGX_ZSBUFFER_DATA              *psSBuffer,
2641                                                                  IMG_BOOL                       bCommitRefCountsTA,
2642                                                                  IMG_BOOL                       bCommitRefCounts3D,
2643                                                                  IMG_BOOL                       *pbCommittedRefCountsTA,
2644                                                                  IMG_BOOL                       *pbCommittedRefCounts3D)
2645 {
2646
2647         IMG_UINT32                              ui32TACmdCount=0;
2648         IMG_UINT32                              ui323DCmdCount=0;
2649         IMG_BOOL                                bKickTADM = IMG_FALSE;
2650         IMG_BOOL                                bKick3DDM = IMG_FALSE;
2651         RGXFWIF_UFO                             sPRUFO;
2652         IMG_UINT32                              *paui32Server3DSyncFlagsPR = IMG_NULL;
2653         IMG_UINT32                              *paui32Server3DSyncFlags3D = IMG_NULL;
2654         IMG_UINT32                              i;
2655         PVRSRV_ERROR                    eError = PVRSRV_OK;
2656         PVRSRV_ERROR                    eError2;
2657
2658         /* Internal client sync info, used to help with merging of Android fd syncs */
2659         IMG_UINT32                              ui32IntClientTAFenceCount = 0;
2660         PRGXFWIF_UFO_ADDR               *pauiIntClientTAFenceUFOAddress = IMG_NULL;
2661         IMG_UINT32                              *paui32IntClientTAFenceValue = IMG_NULL;
2662
2663         IMG_UINT32                              ui32NumUpdateSyncs = 0;
2664         PRGXFWIF_UFO_ADDR               *puiUpdateFWAddrs = IMG_NULL;
2665         IMG_UINT32                              *pui32UpdateValues = IMG_NULL;
2666
2667         PRGXFWIF_UFO_ADDR                       uiPRFenceUFOAddress;
2668
2669 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
2670         /* Android fd sync update info */
2671         struct pvr_sync_append_data *psFDData = NULL;
2672 #endif
2673
2674         PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
2675         PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
2676         PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
2677
2678         PRGXFWIF_UFO_ADDR                       *pauiClientTAFenceUFOAddress;
2679         PRGXFWIF_UFO_ADDR                       *pauiClientTAUpdateUFOAddress;
2680         PRGXFWIF_UFO_ADDR                       *pauiClient3DFenceUFOAddress;
2681         PRGXFWIF_UFO_ADDR                       *pauiClient3DUpdateUFOAddress;
2682
2683         *pbCommittedRefCountsTA = IMG_FALSE;
2684         *pbCommittedRefCounts3D = IMG_FALSE;
2685
2686         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
2687                                                                                 ui32ClientTAFenceCount,
2688                                                                                 apsClientTAFenceSyncPrimBlock,
2689                                                                                 paui32ClientTAFenceSyncOffset);
2690         if(eError != PVRSRV_OK)
2691         {
2692                 goto err_populate_sync_addr_list;
2693         }
2694
2695         pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
2696
2697         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
2698                                                                                 ui32ClientTAUpdateCount,
2699                                                                                 apsClientTAUpdateSyncPrimBlock,
2700                                                                                 paui32ClientTAUpdateSyncOffset);
2701         if(eError != PVRSRV_OK)
2702         {
2703                 goto err_populate_sync_addr_list;
2704         }
2705
2706         pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
2707
2708         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
2709                                                                                 ui32Client3DFenceCount,
2710                                                                                 apsClient3DFenceSyncPrimBlock,
2711                                                                                 paui32Client3DFenceSyncOffset);
2712         if(eError != PVRSRV_OK)
2713         {
2714                 goto err_populate_sync_addr_list;
2715         }
2716
2717         pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
2718
2719         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
2720                                                                                 ui32Client3DUpdateCount,
2721                                                                                 apsClient3DUpdateSyncPrimBlock,
2722                                                                                 paui32Client3DUpdateSyncOffset);
2723         if(eError != PVRSRV_OK)
2724         {
2725                 goto err_populate_sync_addr_list;
2726         }
2727
2728         pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
2729
2730         eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock,
2731                                                                         ui32PRFenceSyncOffset,
2732                                                                         &uiPRFenceUFOAddress);
2733
2734         if(eError != PVRSRV_OK)
2735         {
2736                 goto err_pr_fence_address;
2737         }
2738
2739         /* Sanity check the server fences */
2740         for (i=0;i<ui32ServerTASyncPrims;i++)
2741         {
2742                 if (!(paui32ServerTASyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
2743                 {
2744                         PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on TA) must fence", __FUNCTION__));
2745                         return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
2746                 }
2747         }
2748
2749         for (i=0;i<ui32Server3DSyncPrims;i++)
2750         {
2751                 if (!(paui32Server3DSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK))
2752                 {
2753                         PVR_DPF((PVR_DBG_ERROR, "%s: Server fence (on 3D) must fence", __FUNCTION__));
2754                         return PVRSRV_ERROR_INVALID_SYNC_PRIM_OP;
2755                 }
2756         }
2757
2758         RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
2759                                   & pPreAddr,
2760                                   & pPostAddr,
2761                                   & pRMWUFOAddr);
2762
2763         /*
2764                 Sanity check we have a PR kick if there are client or server fences
2765         */
2766 #if defined(UNDER_WDDM)
2767         /* sanity check only applies to WDDM for a TA kick. 3D kicks don't have a PR kick
2768          * but do have fences
2769          */
2770         if(bKickTA)
2771         {
2772 #endif
2773                 if (!bKickPR & ((ui32Client3DFenceCount != 0) || (ui32Server3DSyncPrims != 0)))
2774                 {
2775                         PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence (client or server) passed without a PR kick", __FUNCTION__));
2776                         return PVRSRV_ERROR_INVALID_PARAMS;
2777                 }
2778 #if defined(UNDER_WDDM)
2779         }
2780 #endif
2781
2782         /* Init and acquire to TA command if required */
2783         if(bKickTA)
2784         {
2785                 RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
2786
2787                 ui32IntClientTAFenceCount = ui32ClientTAFenceCount;
2788                 pauiIntClientTAFenceUFOAddress = pauiClientTAFenceUFOAddress;
2789                 paui32IntClientTAFenceValue = paui32ClientTAFenceValue;
2790
2791 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
2792                 if (ui32NumFenceFDs)
2793                 {
2794                         /*
2795                                 This call is only using the Android fd sync to fence the
2796                                 TA command. There is an update but this is used to
2797                                 indicate that the fence has been finished with and so it
2798                                 can happen after the PR as by then we've finished using
2799                                 the fd sync
2800                         */
2801                         eError =
2802                           pvr_sync_append_fences("TA",
2803                                                  ui32NumFenceFDs,
2804                                                  ai32FenceFDs,
2805                                                                          -1,
2806                                                  ui32NumUpdateSyncs,
2807                                                  puiUpdateFWAddrs,
2808                                                  pui32UpdateValues,
2809                                                  ui32IntClientTAFenceCount,
2810                                                  pauiIntClientTAFenceUFOAddress,
2811                                                  paui32IntClientTAFenceValue,
2812                                                  &psFDData);
2813                         if (eError != PVRSRV_OK)
2814                         {
2815                             goto fail_fdsync;
2816                         }
2817                         ui32NumUpdateSyncs = psFDData->nr_updates;
2818                         puiUpdateFWAddrs = psFDData->update_ufo_addresses;
2819                         pui32UpdateValues = psFDData->update_values;
2820                         ui32IntClientTAFenceCount = psFDData->nr_checks;
2821                         pauiIntClientTAFenceUFOAddress = psFDData->check_ufo_addresses;
2822                         paui32IntClientTAFenceValue = psFDData->check_values;
2823                 }
2824 #endif /* PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC */
2825
2826                 /* Init the TA command helper */
2827                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
2828                                                 ui32IntClientTAFenceCount,
2829                                                 pauiIntClientTAFenceUFOAddress,
2830                                                 paui32IntClientTAFenceValue,
2831                                                 ui32ClientTAUpdateCount,
2832                                                 pauiClientTAUpdateUFOAddress,
2833                                                 paui32ClientTAUpdateValue,
2834                                                 ui32ServerTASyncPrims,
2835                                                 paui32ServerTASyncFlags,
2836                                                 pasServerTASyncs,
2837                                                 ui32TACmdSize,
2838                                                 pui8TADMCmd,
2839                                                 & pPreAddr,
2840                                                 (bKick3D ? IMG_NULL : & pPostAddr),
2841                                                 (bKick3D ? IMG_NULL : & pRMWUFOAddr),
2842                                                 RGXFWIF_CCB_CMD_TYPE_TA,
2843                                                 bPDumpContinuous,
2844                                                 "TA",
2845                                                 &sTACmdHelperData);
2846                 if (eError != PVRSRV_OK)
2847                 {
2848                         goto fail_tacmdinit;
2849                 }
2850
2851                 eError = RGXCmdHelperAcquireCmdCCB(1,
2852                                                                                    &sTACmdHelperData,
2853                                                                                    &bKickTADM);
2854                 if (eError != PVRSRV_OK)
2855                 {
2856                         if (!bKickTADM)
2857                         {
2858                                 goto fail_taacquirecmd;
2859                         }
2860                         else
2861                         {
2862                                 /* commit the TA ref counting next time, when the CCB space is successfully
2863                                  * acquired
2864                                  */
2865                                 bCommitRefCountsTA = IMG_FALSE;
2866                         }
2867                 }
2868                 else
2869                 {
2870                         ui32TACmdCount++;
2871                 }
2872         }
2873
2874         /* Only kick the 3D if required */
2875         if (eError == PVRSRV_OK)
2876         {
2877         if (bKickPR)
2878         {
2879                 RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
2880
2881                 if (ui32Server3DSyncPrims)
2882                 {
2883                         /*
2884                                 The fence (and possible update) straddle multiple commands so
2885                                 we have to modify the flags to do the right things at the right
2886                                 time.
2887                                 At this stage we should only fence, any updates will happen with
2888                                 the normal 3D command.
2889                         */
2890                         paui32Server3DSyncFlagsPR = OSAllocMem(sizeof(IMG_UINT32) * ui32Server3DSyncPrims);
2891                         if (paui32Server3DSyncFlagsPR == IMG_NULL)
2892                         {
2893                                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
2894                                 goto fail_prserversyncflagsallocpr;
2895                         }
2896
2897                         /* Copy only the fence flag across */
2898                         for (i=0;i<ui32Server3DSyncPrims;i++)
2899                         {
2900                                 paui32Server3DSyncFlagsPR[i] = paui32Server3DSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK;
2901                         }
2902                 }
2903
2904                 /*
2905                         The command helper doesn't know about the PR fence so create
2906                         the command with all the fences against it and later create
2907                         the PR command itself which _must_ come after the PR fence.
2908                 */
2909                 sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
2910                 sPRUFO.ui32Value = ui32PRFenceValue;
2911
2912                 /* Init the PR fence command helper */
2913                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
2914                                                                                 ui32Client3DFenceCount,
2915                                                                                 pauiClient3DFenceUFOAddress,
2916                                                                                 paui32Client3DFenceValue,
2917                                                                                 0,
2918                                                                                 IMG_NULL,
2919                                                                                 IMG_NULL,
2920                                                                                 (bKick3D ? ui32Server3DSyncPrims : 0),
2921                                                                                 paui32Server3DSyncFlagsPR,
2922                                                                                 pasServer3DSyncs,
2923                                                                                 sizeof(sPRUFO),
2924                                                                                 (IMG_UINT8*) &sPRUFO,
2925                                                                                 IMG_NULL,
2926                                                                                 IMG_NULL,
2927                                                                                 IMG_NULL,
2928                                                                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
2929                                                                                 bPDumpContinuous,
2930                                                                                 "3D-PR Fence",
2931                                                                                 &as3DCmdHelperData[ui323DCmdCount++]);
2932                 if (eError != PVRSRV_OK)
2933                 {
2934                         goto fail_prfencecmdinit;
2935                 }
2936
2937                 /* Init the 3D PR command helper */
2938                 /*
2939                         See note above PVRFDSyncQueryFencesKM as to why updates for android
2940                         syncs are passed in with the PR
2941                 */
2942                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
2943                                                                                 0,
2944                                                                                 IMG_NULL,
2945                                                                                 IMG_NULL,
2946                                                                                 ui32NumUpdateSyncs,
2947                                                                                 puiUpdateFWAddrs,
2948                                                                                 pui32UpdateValues,
2949                                                                                 0,
2950                                                                                 IMG_NULL,
2951                                                                                 IMG_NULL,
2952                                                                                 ui323DPRCmdSize,
2953                                                                                 pui83DPRDMCmd,
2954                                                                                 IMG_NULL,
2955                                                                                 IMG_NULL,
2956                                                                                 IMG_NULL,
2957                                                                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
2958                                                                                 bPDumpContinuous,
2959                                                                                 "3D-PR",
2960                                                                                 &as3DCmdHelperData[ui323DCmdCount++]);
2961                 if (eError != PVRSRV_OK)
2962                 {
2963                         goto fail_prcmdinit;
2964                 }
2965         }
2966
2967         if (bKick3D || bAbort)
2968         {
2969                 RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
2970
2971                 if (ui32Server3DSyncPrims)
2972                 {
2973                         /*
2974                                 Copy only the update flags for the 3D as the fences will be in
2975                                 the PR command created above
2976                         */
2977                         paui32Server3DSyncFlags3D = OSAllocMem(sizeof(IMG_UINT32) * ui32Server3DSyncPrims);
2978                         if (paui32Server3DSyncFlags3D == IMG_NULL)
2979                         {
2980                                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
2981                                 goto fail_prserversyncflagsalloc3d;
2982                         }
2983
2984                         /* Copy only the update flag across */
2985                         for (i=0;i<ui32Server3DSyncPrims;i++)
2986                         {
2987                                 paui32Server3DSyncFlags3D[i] = paui32Server3DSyncFlags[i] & PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE;
2988                         }
2989                 }
2990
2991                 /* Init the 3D command helper */
2992                 eError = RGXCmdHelperInitCmdCCB(FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
2993                                                 0,
2994                                                 IMG_NULL,
2995                                                 IMG_NULL,
2996                                                 ui32Client3DUpdateCount,
2997                                                 pauiClient3DUpdateUFOAddress,
2998                                                 paui32Client3DUpdateValue,
2999                                                 ui32Server3DSyncPrims,
3000                                                 paui32Server3DSyncFlags3D,
3001                                                 pasServer3DSyncs,
3002                                                 ui323DCmdSize,
3003                                                 pui83DDMCmd,
3004                                                 (bKickTA ? IMG_NULL : & pPreAddr),
3005                                                 & pPostAddr,
3006                                                 & pRMWUFOAddr,
3007                                                 RGXFWIF_CCB_CMD_TYPE_3D,
3008                                                 bPDumpContinuous,
3009                                                 "3D",
3010                                                 &as3DCmdHelperData[ui323DCmdCount++]);
3011                 if (eError != PVRSRV_OK)
3012                 {
3013                         goto fail_3dcmdinit;
3014                 }
3015         }
3016
3017         if (ui323DCmdCount)
3018         {
3019                 PVR_ASSERT(bKickPR || bKick3D);
3020
3021                 /* Acquire space for all the 3D command(s) */
3022                 eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount,
3023                                                                                    as3DCmdHelperData,
3024                                                                                    &bKick3DDM);
3025                 if (eError != PVRSRV_OK)
3026                 {
3027                         if (!bKick3DDM && !bKickTADM)
3028                         {
3029                                 goto fail_3dacquirecmd;
3030                         }
3031                         else
3032                         {
3033                                 /*
3034                                         There are no DM commands but we still need to kick the
3035                                         DM to flush out the padding command.
3036                                         Also reset the TA count as we're going to try again
3037                                 */
3038                                 ui32TACmdCount = 0;
3039                                 ui323DCmdCount = 0;
3040
3041                                 /* commit the 3D ref counting next time, when the CCB space is
3042                                  * successfully acquired
3043                                  */
3044                                 bCommitRefCounts3D = IMG_FALSE;
3045                         }
3046                 }
3047         }
3048         }
3049
3050         /*
3051                 We should acquire the space in the kernel CCB here as after this point
3052                 we release the commands which will take operations on server syncs
3053                 which can't be undone
3054         */
3055
3056         /*
3057                 Everything is ready to go now, release the commands
3058         */
3059         if (ui32TACmdCount)
3060         {
3061                 RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
3062                                                                   &sTACmdHelperData,
3063                                                                   "TA",
3064                                                                   FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
3065         }
3066
3067         if (ui323DCmdCount)
3068         {
3069                 RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
3070                                                                   as3DCmdHelperData,
3071                                                                   "3D",
3072                                                                   FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
3073         }
3074
3075         if (bKickTADM)
3076         {
3077                 RGXFWIF_KCCB_CMD sTAKCCBCmd;
3078
3079                 /* Construct the kernel TA CCB command. */
3080                 sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
3081                 sTAKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
3082                 sTAKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
3083
3084                 if(bCommitRefCountsTA)
3085                 {
3086                         AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTAKCCBCmd.uCmdData.sCmdKickData.apsCleanupCtl,
3087                                                                                 &sTAKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl,
3088                                                                                 RGXFWIF_DM_TA,
3089                                                                                 bKickTA,
3090                                                                                 psRTDataCleanup,
3091                                                                                 psZBuffer,
3092                                                                                 psSBuffer);
3093                         *pbCommittedRefCountsTA = IMG_TRUE;
3094                 }
3095                 else
3096                 {
3097                         sTAKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
3098                 }
3099
3100                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3101                 {
3102                         eError2 = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
3103                                                                                 RGXFWIF_DM_TA,
3104                                                                                 &sTAKCCBCmd,
3105                                                                                 sizeof(sTAKCCBCmd),
3106                                                                                 bPDumpContinuous);
3107                         if (eError2 != PVRSRV_ERROR_RETRY)
3108                         {
3109                                 break;
3110                         }
3111                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3112                 } END_LOOP_UNTIL_TIMEOUT();
3113
3114 #if defined(SUPPORT_GPUTRACE_EVENTS)
3115                 RGXHWPerfFTraceGPUEnqueueEvent(psRenderContext->psDeviceNode->pvDevice,
3116                                 ui32ExtJobRef, ui32IntJobRef, "TA3D");
3117 #endif
3118
3119         }
3120
3121         if (bKick3DDM)
3122         {
3123                 RGXFWIF_KCCB_CMD s3DKCCBCmd;
3124
3125                 /* Construct the kernel 3D CCB command. */
3126                 s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
3127                 s3DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
3128                 s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
3129
3130                 if(bCommitRefCounts3D)
3131                 {
3132                         AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DKCCBCmd.uCmdData.sCmdKickData.apsCleanupCtl,
3133                                                                                         &s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl,
3134                                                                                         RGXFWIF_DM_3D,
3135                                                                                         bKick3D,
3136                                                                                         psRTDataCleanup,
3137                                                                                         psZBuffer,
3138                                                                                         psSBuffer);
3139                         *pbCommittedRefCounts3D = IMG_TRUE;
3140                 }
3141                 else
3142                 {
3143                         s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
3144                 }
3145
3146
3147                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US)
3148                 {
3149                         eError2 = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
3150                                                                                 RGXFWIF_DM_3D,
3151                                                                                 &s3DKCCBCmd,
3152                                                                                 sizeof(s3DKCCBCmd),
3153                                                                                 bPDumpContinuous);
3154                         if (eError2 != PVRSRV_ERROR_RETRY)
3155                         {
3156                                 break;
3157                         }
3158                         OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
3159                 } END_LOOP_UNTIL_TIMEOUT();
3160         }
3161
3162         /*
3163          * Now check eError (which may have returned an error from our earlier calls
3164          * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
3165          * so we check it now...
3166          */
3167         if (eError != PVRSRV_OK )
3168         {
3169                 goto fail_3dacquirecmd;
3170         }
3171
3172 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3173 #if defined(NO_HARDWARE)
3174         pvr_sync_nohw_complete_fences(psFDData);
3175 #endif
3176         pvr_sync_free_append_fences_data(psFDData);
3177
3178 #endif
3179         if(paui32Server3DSyncFlags3D)
3180         {
3181                 OSFreeMem(paui32Server3DSyncFlags3D);
3182         }
3183
3184         if(paui32Server3DSyncFlagsPR)
3185         {
3186                 OSFreeMem(paui32Server3DSyncFlagsPR);
3187         }
3188
3189         return PVRSRV_OK;
3190
3191 fail_3dacquirecmd:
3192 fail_3dcmdinit:
3193         if (paui32Server3DSyncFlags3D)
3194         {
3195                 OSFreeMem(paui32Server3DSyncFlags3D);
3196         }
3197 fail_prserversyncflagsalloc3d:
3198 fail_prcmdinit:
3199 fail_prfencecmdinit:
3200         if (paui32Server3DSyncFlagsPR)
3201         {
3202                 OSFreeMem(paui32Server3DSyncFlagsPR);
3203         }
3204 fail_prserversyncflagsallocpr:
3205 fail_taacquirecmd:
3206 fail_tacmdinit:
3207 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3208         pvr_sync_rollback_append_fences(psFDData);
3209         pvr_sync_free_append_fences_data(psFDData);
3210 fail_fdsync:
3211 #endif
3212 err_pr_fence_address:
3213 err_populate_sync_addr_list:
3214         PVR_ASSERT(eError != PVRSRV_OK);
3215         return eError;
3216 }
3217
3218 PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
3219                                                                                                  RGX_SERVER_RENDER_CONTEXT *psRenderContext,
3220                                                                                                  IMG_UINT32 ui32Priority)
3221 {
3222         PVRSRV_ERROR eError;
3223
3224         if (psRenderContext->sTAData.ui32Priority != ui32Priority)
3225         {
3226                 eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
3227                                                                         psConnection,
3228                                                                         psRenderContext->psDeviceNode->pvDevice,
3229                                                                         ui32Priority,
3230                                                                         RGXFWIF_DM_TA);
3231                 if (eError != PVRSRV_OK)
3232                 {
3233                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the TA part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3234                         goto fail_tacontext;
3235                 }
3236                 psRenderContext->sTAData.ui32Priority = ui32Priority;
3237         }
3238
3239         if (psRenderContext->s3DData.ui32Priority != ui32Priority)
3240         {
3241                 eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
3242                                                                         psConnection,
3243                                                                         psRenderContext->psDeviceNode->pvDevice,
3244                                                                         ui32Priority,
3245                                                                         RGXFWIF_DM_3D);
3246                 if (eError != PVRSRV_OK)
3247                 {
3248                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 3D part of the rendercontext (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3249                         goto fail_3dcontext;
3250                 }
3251                 psRenderContext->s3DData.ui32Priority = ui32Priority;
3252         }
3253         return PVRSRV_OK;
3254
3255 fail_3dcontext:
3256 fail_tacontext:
3257         PVR_ASSERT(eError != PVRSRV_OK);
3258         return eError;
3259 }
3260
3261
3262 /*
3263  * PVRSRVRGXGetLastResetReason
3264  */
3265 PVRSRV_ERROR PVRSRVRGXGetLastRenderContextResetReasonKM(RGX_SERVER_RENDER_CONTEXT       *psRenderContext,
3266                                                         IMG_UINT32 *peLastResetReason)
3267 {
3268         RGX_SERVER_RC_TA_DATA         *psRenderCtxTAData = &(psRenderContext->sTAData);
3269         RGX_SERVER_COMMON_CONTEXT     *psCurrentServerTACommonCtx = psRenderCtxTAData->psServerCommonContext;
3270         RGX_SERVER_RC_3D_DATA         *psRenderCtx3DData = &(psRenderContext->s3DData);
3271         RGX_SERVER_COMMON_CONTEXT     *psCurrentServer3DCommonCtx = psRenderCtx3DData->psServerCommonContext;
3272         RGXFWIF_CONTEXT_RESET_REASON  eLastTAResetReason, eLast3DResetReason;
3273
3274         PVR_ASSERT(psRenderContext != IMG_NULL);
3275         PVR_ASSERT(peLastResetReason != IMG_NULL);
3276
3277         /* Get the last reset reasons from both the TA and 3D so they are reset... */
3278         eLastTAResetReason = FWCommonContextGetLastResetReason(psCurrentServerTACommonCtx);
3279         eLast3DResetReason = FWCommonContextGetLastResetReason(psCurrentServer3DCommonCtx);
3280
3281         /* Combine the reset reason from TA and 3D into one... */
3282         *peLastResetReason = (IMG_UINT32) eLast3DResetReason;
3283         if (eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_NONE  ||
3284             ((eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_INNOCENT_LOCKUP  ||
3285               eLast3DResetReason == RGXFWIF_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING)  &&
3286              (eLastTAResetReason == RGXFWIF_CONTEXT_RESET_REASON_GUILTY_LOCKUP  ||
3287               eLastTAResetReason == RGXFWIF_CONTEXT_RESET_REASON_GUILTY_OVERRUNING)))
3288         {
3289                 *peLastResetReason = (IMG_UINT32) eLastTAResetReason;
3290         }
3291
3292         return PVRSRV_OK;
3293 }
3294
3295
3296 /*
3297  * PVRSRVRGXGetPartialRenderCountKM
3298  */
3299 PVRSRV_ERROR PVRSRVRGXGetPartialRenderCountKM(DEVMEM_MEMDESC *psHWRTDataMemDesc,
3300                                                                                           IMG_UINT32 *pui32NumPartialRenders)
3301 {
3302         RGXFWIF_HWRTDATA *psHWRTData;
3303         PVRSRV_ERROR eError;
3304
3305         eError = DevmemAcquireCpuVirtAddr(psHWRTDataMemDesc, (IMG_VOID **)&psHWRTData);
3306
3307         if (eError != PVRSRV_OK)
3308         {
3309                 PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXGetPartialRenderCountKM: Failed to map Firmware Render Target Data (%u)", eError));
3310                 return eError;
3311         }
3312
3313         *pui32NumPartialRenders = psHWRTData->ui32NumPartialRenders;
3314
3315         DevmemReleaseCpuVirtAddr(psHWRTDataMemDesc);
3316
3317         return PVRSRV_OK;
3318 }
3319
3320
3321 static IMG_BOOL CheckForStalledRenderCtxtCommand(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
3322 {
3323         RGX_SERVER_RENDER_CONTEXT               *psCurrentServerRenderCtx = IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
3324         RGX_SERVER_RC_TA_DATA                   *psRenderCtxTAData = &(psCurrentServerRenderCtx->sTAData);
3325         RGX_SERVER_COMMON_CONTEXT               *psCurrentServerTACommonCtx = psRenderCtxTAData->psServerCommonContext;
3326         RGX_SERVER_RC_3D_DATA                   *psRenderCtx3DData = &(psCurrentServerRenderCtx->s3DData);
3327         RGX_SERVER_COMMON_CONTEXT               *psCurrentServer3DCommonCtx = psRenderCtx3DData->psServerCommonContext;
3328         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf = pvCallbackData;
3329
3330
3331         DumpStalledFWCommonContext(psCurrentServerTACommonCtx, pfnDumpDebugPrintf);
3332         DumpStalledFWCommonContext(psCurrentServer3DCommonCtx, pfnDumpDebugPrintf);
3333
3334         return IMG_TRUE;
3335 }
3336 IMG_VOID CheckForStalledRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo,
3337                                                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf)
3338 {
3339         OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock, DEVINFO_RENDERLIST);
3340         dllist_foreach_node(&(psDevInfo->sRenderCtxtListHead),
3341                                                 CheckForStalledRenderCtxtCommand, pfnDumpDebugPrintf);
3342         OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
3343 }
3344
3345 static IMG_BOOL CheckForStalledClientRenderCtxtCommand(PDLLIST_NODE psNode, IMG_PVOID pvCallbackData)
3346 {
3347         PVRSRV_ERROR *peError = (PVRSRV_ERROR*)pvCallbackData;
3348         RGX_SERVER_RENDER_CONTEXT               *psCurrentServerRenderCtx = IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
3349         RGX_SERVER_RC_TA_DATA                   *psRenderCtxTAData = &(psCurrentServerRenderCtx->sTAData);
3350         RGX_SERVER_COMMON_CONTEXT               *psCurrentServerTACommonCtx = psRenderCtxTAData->psServerCommonContext;
3351         RGX_SERVER_RC_3D_DATA                   *psRenderCtx3DData = &(psCurrentServerRenderCtx->s3DData);
3352         RGX_SERVER_COMMON_CONTEXT               *psCurrentServer3DCommonCtx = psRenderCtx3DData->psServerCommonContext;
3353
3354         if (PVRSRV_ERROR_CCCB_STALLED == CheckStalledClientCommonContext(psCurrentServerTACommonCtx))
3355         {
3356                 *peError = PVRSRV_ERROR_CCCB_STALLED;
3357         }
3358         if (PVRSRV_ERROR_CCCB_STALLED == CheckStalledClientCommonContext(psCurrentServer3DCommonCtx))
3359         {
3360                 *peError = PVRSRV_ERROR_CCCB_STALLED;
3361         }
3362
3363         return IMG_TRUE;
3364 }
3365 IMG_BOOL CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
3366 {
3367         PVRSRV_ERROR eError = PVRSRV_OK;
3368         OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock, DEVINFO_RENDERLIST);
3369         dllist_foreach_node(&(psDevInfo->sRenderCtxtListHead),
3370                                                 CheckForStalledClientRenderCtxtCommand, &eError);
3371         OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
3372         return (PVRSRV_ERROR_CCCB_STALLED == eError)? IMG_TRUE: IMG_FALSE;
3373 }
3374
3375 IMG_EXPORT PVRSRV_ERROR
3376 PVRSRVRGXKickSyncTAKM(RGX_SERVER_RENDER_CONTEXT  *psRenderContext,
3377                        IMG_UINT32                  ui32TAClientFenceCount,
3378                        SYNC_PRIMITIVE_BLOCK                             **apsTAClientFenceSyncPrimBlock,
3379                        IMG_UINT32                                       *paui32TAClientFenceSyncOffset,
3380                        IMG_UINT32                  *paui32TAClientFenceValue,
3381                        IMG_UINT32                  ui32TAClientUpdateCount,
3382                        SYNC_PRIMITIVE_BLOCK                             **apsTAClientUpdateSyncPrimBlock,
3383                        IMG_UINT32                                       *paui32TAClientUpdateSyncOffset,
3384                        IMG_UINT32                  *paui32TAClientUpdateValue,
3385                        IMG_UINT32                  ui32TAServerSyncPrimsCount,
3386                        IMG_UINT32                  *paui32TAServerSyncFlags,
3387                        SERVER_SYNC_PRIMITIVE       **pasTAServerSyncs,
3388                                            IMG_UINT32                  ui323DClientFenceCount,
3389                        SYNC_PRIMITIVE_BLOCK                             **aps3DClientFenceSyncPrimBlock,
3390                        IMG_UINT32                                       *paui323DClientFenceSyncOffset,
3391                                            IMG_UINT32                  *paui323DClientFenceValue,
3392                                            IMG_UINT32                  ui323DClientUpdateCount,
3393                        SYNC_PRIMITIVE_BLOCK                             **aps3DClientUpdateSyncPrimBlock,
3394                        IMG_UINT32                                       *paui323DClientUpdateSyncOffset,
3395                                            IMG_UINT32                  *paui323DClientUpdateValue,
3396                                            IMG_UINT32                  ui323DServerSyncPrimsCount,
3397                                            IMG_UINT32                  *paui323DServerSyncFlags,
3398                                            SERVER_SYNC_PRIMITIVE       **pas3DServerSyncs,
3399                                            IMG_UINT32                              ui32NumFenceFDs,
3400                                            IMG_INT32                               *pai32FenceFDs,
3401                        IMG_BOOL                    bPDumpContinuous)
3402 {
3403         PVRSRV_ERROR eError = PVRSRV_OK;
3404         PRGXFWIF_UFO_ADDR                       *pauiTAClientFenceUFOAddress;
3405         PRGXFWIF_UFO_ADDR                       *pauiTAClientUpdateUFOAddress;
3406         PRGXFWIF_UFO_ADDR                       *paui3DClientFenceUFOAddress;
3407         PRGXFWIF_UFO_ADDR                       *paui3DClientUpdateUFOAddress;
3408
3409 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3410         /* Android fd sync update info */
3411         struct pvr_sync_append_data *psFDData = NULL;
3412 #endif
3413
3414         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
3415                                                                                 ui32TAClientFenceCount,
3416                                                                                 apsTAClientFenceSyncPrimBlock,
3417                                                                                 paui32TAClientFenceSyncOffset);
3418         if(eError != PVRSRV_OK)
3419         {
3420                 goto err_populate_sync_addr_list;
3421         }
3422
3423         pauiTAClientFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
3424
3425         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
3426                                                                                 ui32TAClientUpdateCount,
3427                                                                                 apsTAClientUpdateSyncPrimBlock,
3428                                                                                 paui32TAClientUpdateSyncOffset);
3429         if(eError != PVRSRV_OK)
3430         {
3431                 goto err_populate_sync_addr_list;
3432         }
3433
3434         pauiTAClientUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
3435
3436         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
3437                                                                                 ui323DClientFenceCount,
3438                                                                                 aps3DClientFenceSyncPrimBlock,
3439                                                                                 paui323DClientFenceSyncOffset);
3440         if(eError != PVRSRV_OK)
3441         {
3442                 goto err_populate_sync_addr_list;
3443         }
3444
3445         paui3DClientFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
3446
3447         eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
3448                                                                                 ui323DClientUpdateCount,
3449                                                                                 aps3DClientUpdateSyncPrimBlock,
3450                                                                                 paui323DClientUpdateSyncOffset);
3451         if(eError != PVRSRV_OK)
3452         {
3453                 goto err_populate_sync_addr_list;
3454         }
3455
3456         paui3DClientUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
3457
3458 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3459         /* Android FD fences are hardcoded to updates (IMG_TRUE below), Fences go to the TA and updates to the 3D */
3460         if (ui32NumFenceFDs)
3461         {
3462                 eError =
3463                 pvr_sync_append_fences("TA",
3464                                                                           ui32NumFenceFDs,
3465                                                                           pai32FenceFDs,
3466                                                                           -1,
3467                                                                           ui323DClientUpdateCount,
3468                                                                           paui3DClientUpdateUFOAddress,
3469                                                                           paui323DClientUpdateValue,
3470                                                                           ui323DClientFenceCount,
3471                                                                           paui3DClientFenceUFOAddress,
3472                                                                           paui323DClientFenceValue,
3473                                                                           &psFDData);
3474                 if (eError != PVRSRV_OK)
3475                 {
3476                     goto fail_fdsync;
3477                 }
3478                 ui323DClientUpdateCount = psFDData->nr_updates;
3479                 paui3DClientUpdateUFOAddress = psFDData->update_ufo_addresses;
3480                 paui323DClientUpdateValue = psFDData->update_values;
3481                 ui323DClientFenceCount = psFDData->nr_checks;
3482                 paui3DClientFenceUFOAddress = psFDData->check_ufo_addresses;
3483                 paui323DClientFenceValue = psFDData->check_values;
3484         }
3485 #endif
3486
3487         /* send one command through the TA */
3488         if ((ui32TAClientFenceCount + ui32TAClientUpdateCount + ui32TAServerSyncPrimsCount) > 0)
3489         {
3490                 eError = RGXKickSyncKM(psRenderContext->psDeviceNode,
3491                                      psRenderContext->sTAData.psServerCommonContext,
3492                                      RGXFWIF_DM_TA,
3493                                                          "SyncTA",
3494                                      ui32TAClientFenceCount,
3495                                      pauiTAClientFenceUFOAddress,
3496                                      paui32TAClientFenceValue,
3497                                      ui32TAClientUpdateCount,
3498                                      pauiTAClientUpdateUFOAddress,
3499                                      paui32TAClientUpdateValue,
3500                                      ui32TAServerSyncPrimsCount,
3501                                      paui32TAServerSyncFlags,
3502                                      pasTAServerSyncs,
3503                                      bPDumpContinuous);
3504
3505                 if (eError != PVRSRV_OK)
3506                 {
3507                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send TA sync command (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3508                         goto fail_kickTA;
3509                 }
3510         }
3511
3512         /* and another one through the 3D */
3513         if ((ui323DClientFenceCount + ui323DClientUpdateCount + ui323DServerSyncPrimsCount) > 0)
3514         {
3515                 eError = RGXKickSyncKM(psRenderContext->psDeviceNode,
3516                                      psRenderContext->s3DData.psServerCommonContext,
3517                                      RGXFWIF_DM_3D,
3518                                                          "Sync3D",
3519                                      ui323DClientFenceCount,
3520                                      paui3DClientFenceUFOAddress,
3521                                      paui323DClientFenceValue,
3522                                      ui323DClientUpdateCount,
3523                                      paui3DClientUpdateUFOAddress,
3524                                      paui323DClientUpdateValue,
3525                                      ui323DServerSyncPrimsCount,
3526                                      paui323DServerSyncFlags,
3527                                      pas3DServerSyncs,
3528                                      bPDumpContinuous);
3529
3530                 if (eError != PVRSRV_OK)
3531                 {
3532                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send 3D sync command (%s)", __FUNCTION__, PVRSRVGetErrorStringKM(eError)));
3533                         goto fail_kick3D;
3534                 }
3535         }
3536 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3537 #if defined(NO_HARDWARE)
3538         pvr_sync_nohw_complete_fences(psFDData);
3539 #endif /* NO_HARDWARE */
3540         pvr_sync_free_append_fences_data(psFDData);
3541 #endif /* PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC */
3542
3543         return eError;
3544
3545 fail_kick3D:
3546 fail_kickTA:
3547
3548 #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC)
3549         pvr_sync_rollback_append_fences(psFDData);
3550         pvr_sync_free_append_fences_data(psFDData);
3551 fail_fdsync:
3552 #endif
3553 err_populate_sync_addr_list:
3554         return eError;
3555 }
3556 /******************************************************************************
3557  End of file (rgxta3d.c)
3558 ******************************************************************************/