drivers/gpu/rogue_m/services/shared/common/devicemem.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          Device Memory Management
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Front End (nominally Client side part, but now invokable
   6                 from server too) of device memory management
   7 @License        Dual MIT/GPLv2
   8
   9 The contents of this file are subject to the MIT license as set out below.
  10
  11 Permission is hereby granted, free of charge, to any person obtaining a copy
  12 of this software and associated documentation files (the "Software"), to deal
  13 in the Software without restriction, including without limitation the rights
  14 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  15 copies of the Software, and to permit persons to whom the Software is
  16 furnished to do so, subject to the following conditions:
  17
  18 The above copyright notice and this permission notice shall be included in
  19 all copies or substantial portions of the Software.
  20
  21 Alternatively, the contents of this file may be used under the terms of
  22 the GNU General Public License Version 2 ("GPL") in which case the provisions
  23 of GPL are applicable instead of those above.
  24
  25 If you wish to allow use of your version of this file only under the terms of
  26 GPL, and not to allow others to use your version of this file under the terms
  27 of the MIT license, indicate your decision by deleting the provisions above
  28 and replace them with the notice and other provisions required by GPL as set
  29 out in the file called "GPL-COPYING" included in this distribution. If you do
  30 not delete the provisions above, a recipient may use your version of this file
  31 under the terms of either the MIT license or GPL.
  32
  33 This License is also included in this distribution in the file called
  34 "MIT-COPYING".
  35
  36 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  37 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  38 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  39 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  40 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  41 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  42 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  43 */ /**************************************************************************/
  44
  45 #include "devicemem.h"
  46 #include "img_types.h"
  47 #include "pvr_debug.h"
  48 #include "pvrsrv_error.h"
  49 #include "allocmem.h"
  50 #include "ra.h"
  51 #include "osfunc.h"
  52 #include "devicemem_mmap.h"
  53 #include "devicemem_utils.h"
  54 #include "client_mm_bridge.h"
  55 #if defined(PDUMP)
  56 #include "devicemem_pdump.h"
  57 #endif
  58 #if defined(PVR_RI_DEBUG)
  59 #include "client_ri_bridge.h"
  60 #endif
  61 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
  62 #include "client_devicememhistory_bridge.h"
  63 #endif
  64
  65 #if defined(__KERNEL__)
  66 #include "pvrsrv.h"
  67 #if defined(LINUX)
  68 #include "linux/kernel.h"
  69 #endif
  70 #endif
  71
  72 /** Page size.
  73  *  Should be initialised to the correct value at driver init time.
  74  *  Use macros from devicemem.h to access from outside this module.
  75  */
  76 IMG_UINT32 g_uiLog2PageSize = 0;
  77
  78 static PVRSRV_ERROR
  79 _Mapping_Export(DEVMEM_IMPORT *psImport,
  80                 DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr,
  81                 DEVMEM_EXPORTKEY *puiExportKeyPtr,
  82                 DEVMEM_SIZE_T *puiSize,
  83                 DEVMEM_LOG2ALIGN_T *puiLog2Contig)
  84 {
  85     /* Gets an export handle and key for the PMR used for this mapping */
  86     /* Can only be done if there are no suballocations for this mapping */
  87
  88     PVRSRV_ERROR eError;
  89     DEVMEM_EXPORTHANDLE hPMRExportHandle;
  90     DEVMEM_EXPORTKEY uiExportKey;
  91     IMG_DEVMEM_SIZE_T uiSize;
  92     IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig;
  93
  94     if (psImport == IMG_NULL)
  95     {
  96         eError = PVRSRV_ERROR_INVALID_PARAMS;
  97         goto failParams;
  98     }
  99
 100     if (!psImport->bExportable)
 101     {
 102                 eError = PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
 103         goto failParams;
 104     }
 105
 106     eError = BridgePMRExportPMR(psImport->hBridge,
 107                                 psImport->hPMR,
 108                                 &hPMRExportHandle,
 109                                 &uiSize,
 110                                 &uiLog2Contig,
 111                                 &uiExportKey);
 112     if (eError != PVRSRV_OK)
 113     {
 114         goto failExport;
 115     }
 116
 117     PVR_ASSERT(uiSize == psImport->uiSize);
 118
 119     *phPMRExportHandlePtr = hPMRExportHandle;
 120     *puiExportKeyPtr = uiExportKey;
 121     *puiSize = uiSize;
 122     *puiLog2Contig = uiLog2Contig;
 123
 124     return PVRSRV_OK;
 125
 126     /*
 127       error exit paths follow
 128     */
 129
 130 failExport:
 131 failParams:
 132
 133     PVR_ASSERT(eError != PVRSRV_OK);
 134     return eError;
 135
 136 }
 137
 138 static IMG_VOID
 139 _Mapping_Unexport(DEVMEM_IMPORT *psImport,
 140                   DEVMEM_EXPORTHANDLE hPMRExportHandle)
 141 {
 142     PVRSRV_ERROR eError;
 143
 144     PVR_ASSERT (psImport != IMG_NULL);
 145
 146     eError = BridgePMRUnexportPMR(psImport->hBridge,
 147                                   hPMRExportHandle);
 148     PVR_ASSERT(eError == PVRSRV_OK);
 149 }
 150
 151 /*****************************************************************************
 152  *                    Sub allocation internals                               *
 153  *****************************************************************************/
 154
 155 static PVRSRV_ERROR
 156 _AllocateDeviceMemory(IMG_HANDLE hBridge,
 157                                           IMG_HANDLE hDeviceNode,
 158                                           IMG_UINT32 uiLog2Quantum,
 159                                           IMG_DEVMEM_SIZE_T uiSize,
 160                                           IMG_DEVMEM_SIZE_T uiChunkSize,
 161                                           IMG_UINT32 ui32NumPhysChunks,
 162                                           IMG_UINT32 ui32NumVirtChunks,
 163                                           IMG_BOOL *pabMappingTable,
 164                                           IMG_DEVMEM_ALIGN_T uiAlign,
 165                                           DEVMEM_FLAGS_T uiFlags,
 166                                           IMG_BOOL bExportable,
 167                                           DEVMEM_IMPORT **ppsImport)
 168 {
 169         DEVMEM_IMPORT *psImport;
 170         DEVMEM_FLAGS_T uiPMRFlags;
 171         IMG_HANDLE hPMR;
 172         PVRSRV_ERROR eError;
 173
 174         eError = _DevmemImportStructAlloc(hBridge,
 175                                                                           bExportable,
 176                                                                           &psImport);
 177         if (eError != PVRSRV_OK)
 178         {
 179                 goto failAlloc;
 180         }
 181
 182     /* Check the size is a multiple of the quantum */
 183     PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0);
 184
 185         /* Pass only the PMR flags down */
 186         uiPMRFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK;
 187
 188     eError = BridgePhysmemNewRamBackedPMR(hBridge,
 189                                           hDeviceNode,
 190                                           uiSize,
 191                                           uiChunkSize,
 192                                           ui32NumPhysChunks,
 193                                           ui32NumVirtChunks,
 194                                           pabMappingTable,
 195                                           uiLog2Quantum,
 196                                           uiPMRFlags,
 197                                           &hPMR);
 198     if (eError != PVRSRV_OK)
 199     {
 200         /* Our check above should have ensured this the "not page
 201            multiple" error never happens */
 202         PVR_ASSERT(eError != PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE);
 203
 204         goto failPMR;
 205     }
 206
 207         _DevmemImportStructInit(psImport,
 208                                                         uiSize,
 209                                                         uiAlign,
 210                                                         uiFlags,
 211                                                         hPMR);
 212
 213         *ppsImport = psImport;
 214         return PVRSRV_OK;
 215
 216 failPMR:
 217         _DevmemImportDiscard(psImport);
 218 failAlloc:
 219         PVR_ASSERT(eError != PVRSRV_OK);
 220
 221         return eError;
 222 }
 223
 224 static IMG_VOID
 225 _FreeDeviceMemory(DEVMEM_IMPORT *psImport)
 226 {
 227         _DevmemImportStructRelease(psImport);
 228 }
 229
 230 static IMG_BOOL
 231 _SubAllocImportAlloc(RA_PERARENA_HANDLE hArena,
 232                      RA_LENGTH_T uiSize,
 233                      RA_FLAGS_T _flags,
 234                      /* returned data */
 235                      RA_BASE_T *puiBase,
 236                      RA_LENGTH_T *puiActualSize,
 237                      RA_PERISPAN_HANDLE *phImport)
 238 {
 239     /* When suballocations need a new lump of memory, the RA calls
 240        back here.  Later, in the kernel, we must construct a new PMR
 241        and a pairing between the new lump of virtual memory and the
 242        PMR (whether or not such PMR is backed by physical memory) */
 243     DEVMEM_HEAP *psHeap;
 244     DEVMEM_IMPORT *psImport;
 245     IMG_DEVMEM_ALIGN_T uiAlign;
 246     DEVMEM_FLAGS_T uiFlags;
 247     PVRSRV_ERROR eError;
 248     IMG_BOOL bMappingTable = IMG_TRUE;
 249
 250     uiFlags = (DEVMEM_FLAGS_T) _flags;
 251
 252     /* Per-arena private handle is, for us, the heap */
 253     psHeap = hArena;
 254
 255     /* align to the l.s.b. of the size...  e.g. 96kiB aligned to
 256        32kiB. NB: There is an argument to say that the RA should never
 257        ask us for Non-power-of-2 size anyway, but I don't want to make
 258        that restriction arbitrarily now */
 259     uiAlign = uiSize & ~(uiSize-1);
 260
 261     /* The RA should not have invoked us with a size that is not a
 262        multiple of the quantum anyway */
 263     PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0);
 264
 265         eError = _AllocateDeviceMemory(psHeap->psCtx->hBridge,
 266                                                                    psHeap->psCtx->hDeviceNode,
 267                                                                    psHeap->uiLog2Quantum,
 268                                                                    uiSize,
 269                                                                    uiSize,
 270                                                                    1,
 271                                                                    1,
 272                                                                    &bMappingTable,
 273                                                                    uiAlign,
 274                                                                    uiFlags,
 275                                                                    IMG_FALSE,
 276                                                                    &psImport);
 277         if (eError != PVRSRV_OK)
 278         {
 279                 goto failAlloc;
 280         }
 281
 282 #if defined(PVR_RI_DEBUG)
 283         {
 284                 eError = BridgeRIWritePMREntry (psImport->hBridge,
 285                                                                                 psImport->hPMR,
 286                                                                                 sizeof("PMR sub-allocated"),
 287                                                                                 "PMR sub-allocated",
 288                                                                                 psImport->uiSize);
 289                 if( eError != PVRSRV_OK)
 290                 {
 291                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
 292                 }
 293         }
 294 #endif
 295         /*
 296                 Suballocations always get mapped into the device was we need to
 297                 key the RA off something and as we can't export suballocations
 298                 there is no valid reason to request an allocation an not map it
 299         */
 300         eError = _DevmemImportStructDevMap(psHeap,
 301                                                                            IMG_TRUE,
 302                                                                            psImport);
 303         if (eError != PVRSRV_OK)
 304         {
 305                 goto failMap;
 306         }
 307
 308         *puiBase = psImport->sDeviceImport.sDevVAddr.uiAddr;
 309         *puiActualSize = uiSize;
 310         *phImport = psImport;
 311
 312     return IMG_TRUE;
 313
 314     /*
 315       error exit paths follow
 316     */
 317 failMap:
 318     _FreeDeviceMemory(psImport);
 319 failAlloc:
 320
 321     return IMG_FALSE;
 322 }
 323
 324 static IMG_VOID
 325 _SubAllocImportFree(RA_PERARENA_HANDLE hArena,
 326                     RA_BASE_T uiBase,
 327                     RA_PERISPAN_HANDLE hImport)
 328 {
 329     DEVMEM_IMPORT *psImport = hImport;
 330
 331     PVR_ASSERT(psImport != IMG_NULL);
 332     PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap);
 333     PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr);
 334
 335     _DevmemImportStructDevUnmap(psImport);
 336         _DevmemImportStructRelease(psImport);
 337 }
 338
 339 /*****************************************************************************
 340  *                    Devmem context internals                               *
 341  *****************************************************************************/
 342
 343 static PVRSRV_ERROR
 344 _PopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx,
 345                               IMG_HANDLE hDeviceNode,
 346                               DEVMEM_HEAPCFGID uiHeapBlueprintID)
 347 {
 348     PVRSRV_ERROR eError;
 349     PVRSRV_ERROR eError2;
 350     struct _DEVMEM_HEAP_ **ppsHeapArray;
 351     IMG_UINT32 uiNumHeaps;
 352     IMG_UINT32 uiHeapsToUnwindOnError;
 353     IMG_UINT32 uiHeapIndex;
 354     IMG_DEV_VIRTADDR sDevVAddrBase;
 355     IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH];
 356     IMG_DEVMEM_SIZE_T uiHeapLength;
 357     IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize;
 358     IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment;
 359
 360     eError = DevmemHeapCount(psCtx->hBridge,
 361                              hDeviceNode,
 362                              uiHeapBlueprintID,
 363                              &uiNumHeaps);
 364     if (eError != PVRSRV_OK)
 365     {
 366         goto e0;
 367     }
 368
 369     if (uiNumHeaps == 0)
 370     {
 371         ppsHeapArray = IMG_NULL;
 372     }
 373     else
 374     {
 375         ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps);
 376         if (ppsHeapArray == IMG_NULL)
 377         {
 378             eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 379             goto e0;
 380         }
 381     }
 382
 383     uiHeapsToUnwindOnError = 0;
 384
 385     for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++)
 386     {
 387         eError = DevmemHeapDetails(psCtx->hBridge,
 388                                    hDeviceNode,
 389                                    uiHeapBlueprintID,
 390                                    uiHeapIndex,
 391                                    &aszHeapName[0],
 392                                    sizeof(aszHeapName),
 393                                    &sDevVAddrBase,
 394                                    &uiHeapLength,
 395                                    &uiLog2DataPageSize,
 396                                    &uiLog2ImportAlignment);
 397         if (eError != PVRSRV_OK)
 398         {
 399             goto e1;
 400         }
 401
 402         eError = DevmemCreateHeap(psCtx,
 403                                   sDevVAddrBase,
 404                                   uiHeapLength,
 405                                   uiLog2DataPageSize,
 406                                   uiLog2ImportAlignment,
 407                                   aszHeapName,
 408                                   uiHeapBlueprintID,
 409                                   &ppsHeapArray[uiHeapIndex]);
 410         if (eError != PVRSRV_OK)
 411         {
 412             goto e1;
 413         }
 414
 415         uiHeapsToUnwindOnError = uiHeapIndex + 1;
 416     }
 417
 418     psCtx->uiAutoHeapCount = uiNumHeaps;
 419     psCtx->ppsAutoHeapArray = ppsHeapArray;
 420
 421     PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
 422     PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps);
 423
 424     return PVRSRV_OK;
 425
 426     /*
 427       error exit paths
 428     */
 429  e1:
 430     for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++)
 431     {
 432         eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]);
 433         PVR_ASSERT(eError2 == PVRSRV_OK);
 434     }
 435
 436     if (uiNumHeaps != 0)
 437     {
 438         OSFreeMem(ppsHeapArray);
 439     }
 440
 441  e0:
 442     PVR_ASSERT(eError != PVRSRV_OK);
 443     return eError;
 444 }
 445
 446 static IMG_VOID
 447 _UnpopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx)
 448 {
 449     PVRSRV_ERROR eError2;
 450     IMG_UINT32 uiHeapIndex;
 451     IMG_BOOL bDoCheck = IMG_TRUE;
 452 #if defined(__KERNEL__)
 453     PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
 454     if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
 455     {
 456         bDoCheck = IMG_FALSE;
 457     }
 458 #endif
 459
 460     PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
 461
 462     for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++)
 463     {
 464         eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]);
 465         if (bDoCheck)
 466         {
 467                 PVR_ASSERT(eError2 == PVRSRV_OK);
 468         }
 469     }
 470
 471     if (psCtx->uiAutoHeapCount != 0)
 472     {
 473         OSFreeMem(psCtx->ppsAutoHeapArray);
 474         psCtx->ppsAutoHeapArray = IMG_NULL;
 475     }
 476     psCtx->uiAutoHeapCount = 0;
 477
 478     PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
 479     PVR_ASSERT(psCtx->ppsAutoHeapArray == IMG_NULL);
 480 }
 481
 482
 483 /*****************************************************************************
 484  *                    Devmem context functions                               *
 485  *****************************************************************************/
 486
 487 IMG_INTERNAL PVRSRV_ERROR
 488 DevmemCreateContext(DEVMEM_BRIDGE_HANDLE hBridge,
 489                     IMG_HANDLE hDeviceNode,
 490                     DEVMEM_HEAPCFGID uiHeapBlueprintID,
 491                     DEVMEM_CONTEXT **ppsCtxPtr)
 492 {
 493     PVRSRV_ERROR eError;
 494     DEVMEM_CONTEXT *psCtx;
 495     /* handle to the server-side counterpart of the device memory
 496        context (specifically, for handling mapping to device MMU) */
 497     IMG_HANDLE hDevMemServerContext;
 498     IMG_HANDLE hPrivData;
 499
 500
 501     if (ppsCtxPtr == IMG_NULL)
 502     {
 503         eError = PVRSRV_ERROR_INVALID_PARAMS;
 504         goto e0;
 505     }
 506
 507     psCtx = OSAllocMem(sizeof *psCtx);
 508     if (psCtx == IMG_NULL)
 509     {
 510         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 511         goto e0;
 512     }
 513
 514     psCtx->uiNumHeaps = 0;
 515
 516     psCtx->hBridge = hBridge;
 517
 518     /* Create (server-side) Device Memory context */
 519     eError = BridgeDevmemIntCtxCreate(psCtx->hBridge,
 520                                    hDeviceNode,
 521                                    &hDevMemServerContext,
 522                                    &hPrivData);
 523     if (eError != PVRSRV_OK)
 524     {
 525         goto e1;
 526     }
 527
 528     psCtx->hDeviceNode = hDeviceNode;
 529     psCtx->hDevMemServerContext = hDevMemServerContext;
 530     psCtx->hPrivData = hPrivData;
 531
 532     /* automagic heap creation */
 533     psCtx->uiAutoHeapCount = 0;
 534
 535     eError = _PopulateContextFromBlueprint(psCtx, hDeviceNode, uiHeapBlueprintID);
 536     if (eError != PVRSRV_OK)
 537     {
 538         goto e2;
 539     }
 540
 541
 542     *ppsCtxPtr = psCtx;
 543
 544
 545     PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount);
 546     return PVRSRV_OK;
 547
 548     /*
 549       error exit paths follow
 550     */
 551
 552  e2:
 553     PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
 554     PVR_ASSERT(psCtx->uiNumHeaps == 0);
 555     BridgeDevmemIntCtxDestroy(psCtx->hBridge, hDevMemServerContext);
 556
 557  e1:
 558     OSFreeMem(psCtx);
 559
 560  e0:
 561     PVR_ASSERT(eError != PVRSRV_OK);
 562     return eError;
 563 }
 564
 565 IMG_INTERNAL PVRSRV_ERROR
 566 DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
 567                          IMG_HANDLE *hPrivData)
 568 {
 569         PVRSRV_ERROR eError;
 570
 571         if ((psCtx == IMG_NULL) || (hPrivData == IMG_NULL))
 572         {
 573                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 574                 goto e0;
 575         }
 576
 577         *hPrivData = psCtx->hPrivData;
 578         return PVRSRV_OK;
 579
 580 e0:
 581         PVR_ASSERT(eError != PVRSRV_OK);
 582         return eError;
 583 }
 584
 585 IMG_INTERNAL PVRSRV_ERROR
 586 DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx)
 587 {
 588         PVRSRV_ERROR eError;
 589
 590         if (psCtx == IMG_NULL)
 591         {
 592                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 593                 goto e0;
 594         }
 595         return PVRSRV_OK;
 596
 597 e0:
 598         PVR_ASSERT(eError != PVRSRV_OK);
 599         return eError;
 600 }
 601
 602
 603 IMG_INTERNAL PVRSRV_ERROR
 604 DevmemFindHeapByName(const struct _DEVMEM_CONTEXT_ *psCtx,
 605                      const IMG_CHAR *pszHeapName,
 606                      struct _DEVMEM_HEAP_ **ppsHeapRet)
 607 {
 608     IMG_UINT32 uiHeapIndex;
 609
 610     /* N.B.  This func is only useful for finding "automagic" heaps by name */
 611     for (uiHeapIndex = 0;
 612          uiHeapIndex < psCtx->uiAutoHeapCount;
 613          uiHeapIndex++)
 614     {
 615         if (!OSStringCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName))
 616         {
 617             *ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex];
 618             return PVRSRV_OK;
 619         }
 620     }
 621
 622     return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
 623 }
 624
 625 IMG_INTERNAL PVRSRV_ERROR
 626 DevmemDestroyContext(DEVMEM_CONTEXT *psCtx)
 627 {
 628     PVRSRV_ERROR eError;
 629     IMG_BOOL bDoCheck = IMG_TRUE;
 630
 631 #if defined(__KERNEL__)
 632     PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
 633     if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
 634     {
 635         bDoCheck = IMG_FALSE;
 636     }
 637 #endif
 638
 639     if (psCtx == IMG_NULL)
 640     {
 641         return PVRSRV_ERROR_INVALID_PARAMS;
 642     }
 643
 644     /* should be only the automagically instantiated heaps left */
 645     if (psCtx->uiNumHeaps != psCtx->uiAutoHeapCount)
 646     {
 647         return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
 648     }
 649
 650     _UnpopulateContextFromBlueprint(psCtx);
 651
 652     if (bDoCheck)
 653     {
 654                 PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
 655                 PVR_ASSERT(psCtx->uiNumHeaps == 0);
 656     }
 657     eError = BridgeDevmemIntCtxDestroy(psCtx->hBridge,
 658                                       psCtx->hDevMemServerContext);
 659     if (bDoCheck)
 660     {
 661         PVR_ASSERT (eError == PVRSRV_OK);
 662     }
 663
 664     OSFreeMem(psCtx);
 665
 666     return PVRSRV_OK;
 667 }
 668
 669 /*****************************************************************************
 670  *                 Devmem heap query functions                               *
 671  *****************************************************************************/
 672
 673 IMG_INTERNAL PVRSRV_ERROR
 674 DevmemHeapConfigCount(DEVMEM_BRIDGE_HANDLE hBridge,
 675                       IMG_HANDLE hDeviceNode,
 676                       IMG_UINT32 *puiNumHeapConfigsOut)
 677 {
 678     PVRSRV_ERROR eError;
 679
 680     eError = BridgeHeapCfgHeapConfigCount(hBridge,
 681                                           hDeviceNode,
 682                                           puiNumHeapConfigsOut);
 683     return eError;
 684 }
 685
 686 IMG_INTERNAL PVRSRV_ERROR
 687 DevmemHeapCount(DEVMEM_BRIDGE_HANDLE hBridge,
 688                 IMG_HANDLE hDeviceNode,
 689                 IMG_UINT32 uiHeapConfigIndex,
 690                 IMG_UINT32 *puiNumHeapsOut)
 691 {
 692     PVRSRV_ERROR eError;
 693
 694     eError = BridgeHeapCfgHeapCount(hBridge,
 695                                     hDeviceNode,
 696                                     uiHeapConfigIndex,
 697                                     puiNumHeapsOut);
 698     return eError;
 699 }
 700
 701 IMG_INTERNAL PVRSRV_ERROR
 702 DevmemHeapConfigName(DEVMEM_BRIDGE_HANDLE hBridge,
 703                      IMG_HANDLE hDeviceNode,
 704                      IMG_UINT32 uiHeapConfigIndex,
 705                      IMG_CHAR *pszConfigNameOut,
 706                      IMG_UINT32 uiConfigNameBufSz)
 707 {
 708     PVRSRV_ERROR eError;
 709
 710     eError = BridgeHeapCfgHeapConfigName(hBridge,
 711                                          hDeviceNode,
 712                                          uiHeapConfigIndex,
 713                                          uiConfigNameBufSz,
 714                                          pszConfigNameOut);
 715     return eError;
 716 }
 717
 718 IMG_INTERNAL PVRSRV_ERROR
 719 DevmemHeapDetails(DEVMEM_BRIDGE_HANDLE hBridge,
 720                   IMG_HANDLE hDeviceNode,
 721                   IMG_UINT32 uiHeapConfigIndex,
 722                   IMG_UINT32 uiHeapIndex,
 723                   IMG_CHAR *pszHeapNameOut,
 724                   IMG_UINT32 uiHeapNameBufSz,
 725                   IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
 726                   IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
 727                   IMG_UINT32 *puiLog2DataPageSizeOut,
 728                   IMG_UINT32 *puiLog2ImportAlignmentOut)
 729 {
 730     PVRSRV_ERROR eError;
 731
 732     eError = BridgeHeapCfgHeapDetails(hBridge,
 733                                       hDeviceNode,
 734                                       uiHeapConfigIndex,
 735                                       uiHeapIndex,
 736                                       uiHeapNameBufSz,
 737                                       pszHeapNameOut,
 738                                       psDevVAddrBaseOut,
 739                                       puiHeapLengthOut,
 740                                       puiLog2DataPageSizeOut,
 741                                       puiLog2ImportAlignmentOut);
 742
 743     VG_MARK_INITIALIZED(pszHeapNameOut,uiHeapNameBufSz);
 744
 745     return eError;
 746 }
 747
 748 /*****************************************************************************
 749  *                    Devmem heap functions                                  *
 750  *****************************************************************************/
 751
 752 /* See devicemem.h for important notes regarding the arguments
 753    to this function */
 754 IMG_INTERNAL PVRSRV_ERROR
 755 DevmemCreateHeap(DEVMEM_CONTEXT *psCtx,
 756                  IMG_DEV_VIRTADDR sBaseAddress,
 757                  IMG_DEVMEM_SIZE_T uiLength,
 758                  IMG_UINT32 ui32Log2Quantum,
 759                  IMG_UINT32 ui32Log2ImportAlignment,
 760                  const IMG_CHAR *pszName,
 761                  DEVMEM_HEAPCFGID uiHeapBlueprintID,
 762                  DEVMEM_HEAP **ppsHeapPtr)
 763 {
 764     PVRSRV_ERROR eError = PVRSRV_OK;
 765     PVRSRV_ERROR eError2;
 766     DEVMEM_HEAP *psHeap;
 767     /* handle to the server-side counterpart of the device memory
 768        heap (specifically, for handling mapping to device MMU */
 769     IMG_HANDLE hDevMemServerHeap;
 770     IMG_BOOL bRANoSplit = IMG_FALSE;
 771
 772     IMG_CHAR aszBuf[100];
 773     IMG_CHAR *pszStr;
 774
 775     if (ppsHeapPtr == IMG_NULL)
 776     {
 777         eError = PVRSRV_ERROR_INVALID_PARAMS;
 778         goto e0;
 779     }
 780
 781     psHeap = OSAllocMem(sizeof *psHeap);
 782     if (psHeap == IMG_NULL)
 783     {
 784         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 785         goto e0;
 786     }
 787
 788     /* Need to keep local copy of heap name, so caller may free
 789        theirs */
 790     pszStr = OSAllocMem(OSStringLength(pszName)+1);
 791     if (pszStr == IMG_NULL)
 792     {
 793                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 794         goto e1;
 795     }
 796     OSStringCopy(pszStr, pszName);
 797     psHeap->pszName = pszStr;
 798
 799     psHeap->sBaseAddress = sBaseAddress;
 800     OSAtomicWrite(&psHeap->hImportCount,0);
 801
 802     OSSNPrintf(aszBuf, sizeof(aszBuf),
 803                "NDM heap '%s' (suballocs) ctx:%p",
 804                pszName, psCtx);
 805     pszStr = OSAllocMem(OSStringLength(aszBuf)+1);
 806     if (pszStr == IMG_NULL)
 807     {
 808                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 809         goto e2;
 810     }
 811     OSStringCopy(pszStr, aszBuf);
 812     psHeap->pszSubAllocRAName = pszStr;
 813
 814 #if defined(PDUMP) && defined(ANDROID)
 815     /* the META heap is shared globally so a single
 816      * physical memory import may be used to satisfy
 817      * allocations of different processes.
 818      * This is problematic when PDumping because the
 819      * physical memory import used to satisfy a new allocation
 820      * may actually have been imported (and thus the PDump MALLOC
 821      * generated) before the PDump client was started, leading to the
 822      * MALLOC being missing.
 823      * This is solved by disabling splitting of imports for the META physmem
 824      * RA, meaning that every firmware allocation gets its own import, thus
 825      * ensuring the MALLOC is present for every allocation made within the
 826      * pdump capture range
 827      */
 828     if(uiHeapBlueprintID == DEVMEM_HEAPCFG_META)
 829     {
 830         bRANoSplit = IMG_TRUE;
 831     }
 832 #else
 833     PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID);
 834 #endif
 835
 836
 837     psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName,
 838                        /* Subsequent imports: */
 839                        ui32Log2Quantum,
 840                                            RA_LOCKCLASS_2,
 841                        _SubAllocImportAlloc,
 842                        _SubAllocImportFree,
 843                        (RA_PERARENA_HANDLE) psHeap,
 844                        bRANoSplit);
 845     if (psHeap->psSubAllocRA == IMG_NULL)
 846     {
 847         eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
 848         goto e3;
 849     }
 850
 851     psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment;
 852     psHeap->uiLog2Quantum = ui32Log2Quantum;
 853
 854     OSSNPrintf(aszBuf, sizeof(aszBuf),
 855                "NDM heap '%s' (QVM) ctx:%p",
 856                pszName, psCtx);
 857     pszStr = OSAllocMem(OSStringLength(aszBuf)+1);
 858     if (pszStr == IMG_NULL)
 859     {
 860                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 861         goto e4;
 862     }
 863     OSStringCopy(pszStr, aszBuf);
 864     psHeap->pszQuantizedVMRAName = pszStr;
 865
 866     psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName,
 867                        /* Subsequent import: */
 868                                        0, RA_LOCKCLASS_1, IMG_NULL, IMG_NULL,
 869                        (RA_PERARENA_HANDLE) psHeap,
 870                        IMG_FALSE);
 871
 872     if (psHeap->psQuantizedVMRA == IMG_NULL)
 873     {
 874         eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
 875         goto e5;
 876     }
 877
 878         if (!RA_Add(psHeap->psQuantizedVMRA,
 879                        (RA_BASE_T)sBaseAddress.uiAddr,
 880                        (RA_LENGTH_T)uiLength,
 881                        (RA_FLAGS_T)0, /* This RA doesn't use or need flags */
 882                                 IMG_NULL /* per ispan handle */))
 883         {
 884                 RA_Delete(psHeap->psQuantizedVMRA);
 885         eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
 886         goto e5;
 887         }
 888
 889
 890     psHeap->psCtx = psCtx;
 891
 892
 893     /* Create server-side counterpart of Device Memory heap */
 894     eError = BridgeDevmemIntHeapCreate(psCtx->hBridge,
 895                                       psCtx->hDevMemServerContext,
 896                                       sBaseAddress,
 897                                       uiLength,
 898                                       ui32Log2Quantum,
 899                                       &hDevMemServerHeap);
 900     if (eError != PVRSRV_OK)
 901     {
 902         goto e6;
 903     }
 904     psHeap->hDevMemServerHeap = hDevMemServerHeap;
 905
 906         eError = OSLockCreate(&psHeap->hLock, LOCK_TYPE_PASSIVE);
 907         if (eError != PVRSRV_OK)
 908         {
 909                 goto e7;
 910         }
 911
 912     psHeap->psCtx->uiNumHeaps ++;
 913     *ppsHeapPtr = psHeap;
 914
 915 #if defined PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING
 916     psHeap->psMemDescList = IMG_NULL;
 917 #endif  /* PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING */
 918
 919     return PVRSRV_OK;
 920
 921     /*
 922       error exit paths
 923     */
 924  e7:
 925     eError2 = BridgeDevmemIntHeapDestroy(psCtx->hBridge,
 926                                        psHeap->hDevMemServerHeap);
 927     PVR_ASSERT (eError2 == PVRSRV_OK);
 928  e6:
 929     RA_Delete(psHeap->psQuantizedVMRA);
 930  e5:
 931     OSFreeMem(psHeap->pszQuantizedVMRAName);
 932  e4:
 933     RA_Delete(psHeap->psSubAllocRA);
 934  e3:
 935     OSFreeMem(psHeap->pszSubAllocRAName);
 936  e2:
 937     OSFreeMem(psHeap->pszName);
 938  e1:
 939     OSFreeMem(psHeap);
 940  e0:
 941     PVR_ASSERT(eError != PVRSRV_OK);
 942     return eError;
 943 }
 944
 945 IMG_INTERNAL PVRSRV_ERROR
 946 DevmemGetHeapBaseDevVAddr(struct _DEVMEM_HEAP_ *psHeap,
 947                           IMG_DEV_VIRTADDR *pDevVAddr)
 948 {
 949         if (psHeap == IMG_NULL)
 950         {
 951                 return PVRSRV_ERROR_INVALID_PARAMS;
 952         }
 953
 954         *pDevVAddr = psHeap->sBaseAddress;
 955
 956         return PVRSRV_OK;
 957 }
 958
 959 IMG_INTERNAL IMG_VOID
 960 DevmemExportalignAdjustSizeAndAlign(DEVMEM_HEAP *psHeap, IMG_DEVMEM_SIZE_T *puiSize, IMG_DEVMEM_ALIGN_T *puiAlign)
 961 {
 962         IMG_DEVMEM_SIZE_T uiSize = *puiSize;
 963         IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign;
 964         IMG_UINT32 uiLog2Quantum;
 965
 966         if (psHeap)
 967         {
 968                 uiLog2Quantum = psHeap->uiLog2Quantum;
 969         }
 970         else
 971         {
 972                 uiLog2Quantum = GET_LOG2_PAGESIZE();
 973         }
 974
 975     if ((1ULL << uiLog2Quantum) > uiAlign)
 976     {
 977                 uiAlign = 1ULL << uiLog2Quantum;
 978     }
 979     uiSize = (uiSize + uiAlign - 1) & ~(uiAlign - 1);
 980
 981         *puiSize = uiSize;
 982         *puiAlign = uiAlign;
 983 }
 984
 985
 986 IMG_INTERNAL PVRSRV_ERROR
 987 DevmemDestroyHeap(DEVMEM_HEAP *psHeap)
 988 {
 989     PVRSRV_ERROR eError;
 990         IMG_INT uiImportCount;
 991
 992     if (psHeap == IMG_NULL)
 993     {
 994         return PVRSRV_ERROR_INVALID_PARAMS;
 995     }
 996
 997         uiImportCount = OSAtomicRead(&psHeap->hImportCount);
 998     if (uiImportCount > 0)
 999     {
1000         PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName));
1001         return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
1002     }
1003
1004         OSLockDestroy(psHeap->hLock);
1005
1006     PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0);
1007     psHeap->psCtx->uiNumHeaps --;
1008
1009     eError = BridgeDevmemIntHeapDestroy(psHeap->psCtx->hBridge,
1010                                        psHeap->hDevMemServerHeap);
1011     PVR_ASSERT (eError == PVRSRV_OK);
1012
1013     RA_Delete(psHeap->psQuantizedVMRA);
1014     OSFreeMem(psHeap->pszQuantizedVMRAName);
1015
1016     RA_Delete(psHeap->psSubAllocRA);
1017     OSFreeMem(psHeap->pszSubAllocRAName);
1018
1019     OSFreeMem(psHeap->pszName);
1020
1021     OSFreeMem(psHeap);
1022
1023     return PVRSRV_OK;
1024 }
1025
1026 /*****************************************************************************
1027  *                Devmem allocation/free functions                           *
1028  *****************************************************************************/
1029
1030 IMG_INTERNAL PVRSRV_ERROR
1031 DevmemAllocate(DEVMEM_HEAP *psHeap,
1032                IMG_DEVMEM_SIZE_T uiSize,
1033                IMG_DEVMEM_ALIGN_T uiAlign,
1034                DEVMEM_FLAGS_T uiFlags,
1035                const IMG_PCHAR pszText,
1036                            DEVMEM_MEMDESC **ppsMemDescPtr)
1037 {
1038     IMG_BOOL bStatus; /* eError for RA */
1039     RA_BASE_T uiAllocatedAddr;
1040     RA_LENGTH_T uiAllocatedSize;
1041     RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */
1042     RA_FLAGS_T uiFlagsForRA;
1043     PVRSRV_ERROR eError;
1044     DEVMEM_MEMDESC *psMemDesc = IMG_NULL;
1045         IMG_DEVMEM_OFFSET_T uiOffset = 0;
1046         DEVMEM_IMPORT *psImport;
1047         IMG_VOID *pvAddr;
1048
1049         if (uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC)
1050         {
1051                 /* Deferred Allocation not supported on SubAllocs*/
1052                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1053                 goto failParams;
1054         }
1055
1056     if (psHeap == IMG_NULL || ppsMemDescPtr == IMG_NULL)
1057     {
1058         eError = PVRSRV_ERROR_INVALID_PARAMS;
1059         goto failParams;
1060     }
1061
1062         eError = _DevmemValidateParams(uiSize,
1063                                                                    uiAlign,
1064                                                                    uiFlags);
1065         if (eError != PVRSRV_OK)
1066         {
1067                 goto failParams;
1068         }
1069
1070         eError =_DevmemMemDescAlloc(&psMemDesc);
1071     if (eError != PVRSRV_OK)
1072     {
1073         goto failMemDescAlloc;
1074     }
1075
1076     /*
1077         If zero flag is set we have to have write access to the page.
1078     */
1079     uiFlags |= (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0;
1080
1081         /*
1082                 No request for exportable memory so use the RA
1083         */
1084     uiFlagsForRA = (RA_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
1085     /* Check that the cast didn't lose any flags due to different integer widths */
1086     PVR_ASSERT(uiFlagsForRA == (uiFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK));
1087
1088         /*
1089            When the RA suballocates memory from a Span it does not zero it. It only zeroes the
1090            memory if it allocates a new Span; but we don't know what is going to happen for this
1091            RA_Alloc call. Therefore, we zero the mem after the allocation below.
1092         */
1093         uiFlagsForRA &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
1094
1095         bStatus = RA_Alloc(psHeap->psSubAllocRA,
1096                                            uiSize,
1097                                            uiFlagsForRA,
1098                                            uiAlign,
1099                                            &uiAllocatedAddr,
1100                                            &uiAllocatedSize,
1101                                            &hImport);
1102         if (!bStatus)
1103         {
1104                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1105                 goto failDeviceMemAlloc;
1106         }
1107
1108         psImport = hImport;
1109         uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr;
1110
1111         _DevmemMemDescInit(psMemDesc,
1112                                            uiOffset,
1113                                            psImport);
1114
1115         /* zero the memory */
1116         if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
1117
1118         {
1119                 eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
1120                 if (eError != PVRSRV_OK)
1121                 {
1122                         goto failZero;
1123                 }
1124
1125                 /* FIXME: uiSize is a 64-bit quantity whereas the 3rd argument
1126                  * to OSDeviceMemSet is a 32-bit quantity on 32-bit systems
1127                  * hence a compiler warning of implicit cast and loss of data.
1128                  * Added explicit cast and assert to remove warning.
1129                  */
1130 #if (defined(_WIN32) && !defined(_WIN64)) || (defined(LINUX) && defined(__i386__))
1131                 PVR_ASSERT(uiSize<IMG_UINT32_MAX);
1132 #endif
1133
1134                 OSDeviceMemSet(pvAddr, 0x0, (IMG_SIZE_T) uiSize);
1135
1136                 DevmemReleaseCpuVirtAddr(psMemDesc);
1137
1138 #if defined(PDUMP)
1139                 DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS);
1140 #endif
1141         }
1142
1143 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
1144         /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
1145          * the allocation gets mapped/unmapped
1146          */
1147         OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
1148         psMemDesc->sTraceData.uiSize = uiSize;
1149 #endif
1150
1151 #if defined(PVR_RI_DEBUG)
1152         {
1153                 /* Attach RI information */
1154                 eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hBridge,
1155                                                                                         psMemDesc->psImport->hPMR,
1156                                                                                         OSStringNLength(pszText, RI_MAX_TEXT_LEN),
1157                                                                                         pszText,
1158                                                                                         psMemDesc->uiOffset,
1159                                                                                         uiAllocatedSize,
1160                                                                                         IMG_FALSE,
1161                                                                                         IMG_FALSE,
1162                                                                                         &(psMemDesc->hRIHandle));
1163                 if( eError != PVRSRV_OK)
1164                 {
1165                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
1166                 }
1167         }
1168 #else  /* if defined(PVR_RI_DEBUG) */
1169         PVR_UNREFERENCED_PARAMETER (pszText);
1170 #endif /* if defined(PVR_RI_DEBUG) */
1171
1172         *ppsMemDescPtr = psMemDesc;
1173
1174     return PVRSRV_OK;
1175
1176     /*
1177       error exit paths follow
1178     */
1179
1180 failZero:
1181         _DevmemMemDescRelease(psMemDesc);
1182         psMemDesc = IMG_NULL;   /* Make sure we don't do a discard after the release */
1183 failDeviceMemAlloc:
1184         if (psMemDesc)
1185                 _DevmemMemDescDiscard(psMemDesc);
1186 failMemDescAlloc:
1187 failParams:
1188     PVR_ASSERT(eError != PVRSRV_OK);
1189     return eError;
1190 }
1191
1192
1193
1194 IMG_INTERNAL PVRSRV_ERROR
1195 DevmemAllocateExportable(IMG_HANDLE hBridge,
1196                                                  IMG_HANDLE hDeviceNode,
1197                                                  IMG_DEVMEM_SIZE_T uiSize,
1198                                                  IMG_DEVMEM_ALIGN_T uiAlign,
1199                                                  DEVMEM_FLAGS_T uiFlags,
1200                                                  const IMG_PCHAR pszText,
1201                                                  DEVMEM_MEMDESC **ppsMemDescPtr)
1202 {
1203     PVRSRV_ERROR eError;
1204     DEVMEM_MEMDESC *psMemDesc = IMG_NULL;
1205         DEVMEM_IMPORT *psImport;
1206         IMG_BOOL bMappingTable = IMG_TRUE;
1207
1208
1209         DevmemExportalignAdjustSizeAndAlign(IMG_NULL,
1210                                                                                 &uiSize,
1211                                                                                 &uiAlign);
1212
1213         eError = _DevmemValidateParams(uiSize,
1214                                                                    uiAlign,
1215                                                                    uiFlags);
1216         if (eError != PVRSRV_OK)
1217         {
1218                 goto failParams;
1219         }
1220
1221
1222         eError =_DevmemMemDescAlloc(&psMemDesc);
1223     if (eError != PVRSRV_OK)
1224     {
1225         goto failMemDescAlloc;
1226     }
1227
1228         /*
1229                 Note:
1230                 In the case of exportable memory we have no heap to
1231                 query the pagesize from, so we assume host pagesize.
1232         */
1233         eError = _AllocateDeviceMemory(hBridge,
1234                                                                    hDeviceNode,
1235                                                                    GET_LOG2_PAGESIZE(),
1236                                                                    uiSize,
1237                                                                    uiSize,
1238                                                                    1,
1239                                                                    1,
1240                                                                    &bMappingTable,
1241                                                                    uiAlign,
1242                                                                    uiFlags,
1243                                                                    IMG_TRUE,
1244                                                                    &psImport);
1245         if (eError != PVRSRV_OK)
1246         {
1247                 goto failDeviceMemAlloc;
1248         }
1249
1250         _DevmemMemDescInit(psMemDesc,
1251                                            0,
1252                                            psImport);
1253
1254     *ppsMemDescPtr = psMemDesc;
1255
1256 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
1257         /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
1258          * the allocation gets mapped/unmapped
1259          */
1260         OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
1261         psMemDesc->sTraceData.uiSize = uiSize;
1262 #endif
1263
1264 #if defined(PVR_RI_DEBUG)
1265         {
1266                 eError = BridgeRIWritePMREntry (psImport->hBridge,
1267                                                                                 psImport->hPMR,
1268                                                                                 OSStringNLength(pszText, RI_MAX_TEXT_LEN),
1269                                                                                 (IMG_CHAR *)pszText,
1270                                                                                 psImport->uiSize);
1271                 if( eError != PVRSRV_OK)
1272                 {
1273                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
1274                 }
1275
1276                  /* Attach RI information */
1277                 eError = BridgeRIWriteMEMDESCEntry (psImport->hBridge,
1278                                                                                         psImport->hPMR,
1279                                                                                         sizeof("^"),
1280                                                                                         "^",
1281                                                                                         psMemDesc->uiOffset,
1282                                                                                         uiSize,
1283                                                                                         IMG_FALSE,
1284                                                                                         IMG_TRUE,
1285                                                                                         &psMemDesc->hRIHandle);
1286                 if( eError != PVRSRV_OK)
1287                 {
1288                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
1289                 }
1290         }
1291 #else  /* if defined(PVR_RI_DEBUG) */
1292         PVR_UNREFERENCED_PARAMETER (pszText);
1293 #endif /* if defined(PVR_RI_DEBUG) */
1294
1295     return PVRSRV_OK;
1296
1297     /*
1298       error exit paths follow
1299     */
1300
1301 failDeviceMemAlloc:
1302     _DevmemMemDescDiscard(psMemDesc);
1303
1304 failMemDescAlloc:
1305 failParams:
1306     PVR_ASSERT(eError != PVRSRV_OK);
1307     return eError;
1308 }
1309
1310 IMG_INTERNAL PVRSRV_ERROR
1311 DevmemAllocateSparse(IMG_HANDLE hBridge,
1312                                          IMG_HANDLE hDeviceNode,
1313                                          IMG_DEVMEM_SIZE_T uiSize,
1314                                          IMG_DEVMEM_SIZE_T uiChunkSize,
1315                                          IMG_UINT32 ui32NumPhysChunks,
1316                                          IMG_UINT32 ui32NumVirtChunks,
1317                                          IMG_BOOL *pabMappingTable,
1318                                          IMG_DEVMEM_ALIGN_T uiAlign,
1319                                          DEVMEM_FLAGS_T uiFlags,
1320                                          const IMG_PCHAR pszText,
1321                                          DEVMEM_MEMDESC **ppsMemDescPtr)
1322 {
1323     PVRSRV_ERROR eError;
1324     DEVMEM_MEMDESC *psMemDesc = IMG_NULL;
1325         DEVMEM_IMPORT *psImport;
1326
1327
1328         DevmemExportalignAdjustSizeAndAlign(IMG_NULL,
1329                                                                                 &uiSize,
1330                                                                                 &uiAlign);
1331
1332         eError = _DevmemValidateParams(uiSize,
1333                                                                    uiAlign,
1334                                                                    uiFlags);
1335         if (eError != PVRSRV_OK)
1336         {
1337                 goto failParams;
1338         }
1339
1340
1341         eError =_DevmemMemDescAlloc(&psMemDesc);
1342     if (eError != PVRSRV_OK)
1343     {
1344         goto failMemDescAlloc;
1345     }
1346
1347         /*
1348                 Note:
1349                 In the case of sparse memory we have no heap to
1350                 query the pagesize from, so we assume host pagesize.
1351         */
1352         eError = _AllocateDeviceMemory(hBridge,
1353                                                                    hDeviceNode,
1354                                                                    GET_LOG2_PAGESIZE(),
1355                                                                    uiSize,
1356                                                                    uiChunkSize,
1357                                                                    ui32NumPhysChunks,
1358                                                                    ui32NumVirtChunks,
1359                                                                    pabMappingTable,
1360                                                                    uiAlign,
1361                                                                    uiFlags,
1362                                                                    IMG_TRUE,
1363                                                                    &psImport);
1364         if (eError != PVRSRV_OK)
1365         {
1366                 goto failDeviceMemAlloc;
1367         }
1368
1369         _DevmemMemDescInit(psMemDesc,
1370                                            0,
1371                                            psImport);
1372
1373 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
1374         /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
1375          * the allocation gets mapped/unmapped
1376          */
1377         OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
1378         psMemDesc->sTraceData.uiSize = uiSize;
1379 #endif
1380
1381 #if defined(PVR_RI_DEBUG)
1382         {
1383                 eError = BridgeRIWritePMREntry (psImport->hBridge,
1384                                                                                 psImport->hPMR,
1385                                                                                 OSStringNLength(pszText, RI_MAX_TEXT_LEN),
1386                                                                                 (IMG_CHAR *)pszText,
1387                                                                                 psImport->uiSize);
1388                 if( eError != PVRSRV_OK)
1389                 {
1390                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
1391                 }
1392
1393                 /* Attach RI information */
1394         eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hBridge,
1395                                                                                         psMemDesc->psImport->hPMR,
1396                                                                                         sizeof("^"),
1397                                                                                         "^",
1398                                                                                         psMemDesc->uiOffset,
1399                                                                                         uiSize,
1400                                                                                         IMG_FALSE,
1401                                                                                         IMG_TRUE,
1402                                                                                         &psMemDesc->hRIHandle);
1403                 if( eError != PVRSRV_OK)
1404                 {
1405                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
1406                 }
1407         }
1408 #else  /* if defined(PVR_RI_DEBUG) */
1409         PVR_UNREFERENCED_PARAMETER (pszText);
1410 #endif /* if defined(PVR_RI_DEBUG) */
1411
1412         *ppsMemDescPtr = psMemDesc;
1413
1414     return PVRSRV_OK;
1415
1416     /*
1417       error exit paths follow
1418     */
1419
1420 failDeviceMemAlloc:
1421     _DevmemMemDescDiscard(psMemDesc);
1422
1423 failMemDescAlloc:
1424 failParams:
1425     PVR_ASSERT(eError != PVRSRV_OK);
1426     return eError;
1427 }
1428
1429 /*****************************************************************************
1430  *                Devmem unsecure export functions                           *
1431  *****************************************************************************/
1432
1433 IMG_INTERNAL PVRSRV_ERROR
1434 DevmemExport(DEVMEM_MEMDESC *psMemDesc,
1435              DEVMEM_EXPORTCOOKIE *psExportCookie)
1436 {
1437     /* Caller to provide storage for export cookie struct */
1438     PVRSRV_ERROR eError;
1439     IMG_HANDLE hPMRExportHandle = 0;
1440     IMG_UINT64 uiPMRExportPassword = 0;
1441     IMG_DEVMEM_SIZE_T uiSize = 0;
1442     IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0;
1443
1444     if (psMemDesc == IMG_NULL || psExportCookie == IMG_NULL)
1445     {
1446         eError = PVRSRV_ERROR_INVALID_PARAMS;
1447         goto e0;
1448     }
1449
1450     eError = _Mapping_Export(psMemDesc->psImport,
1451                              &hPMRExportHandle,
1452                              &uiPMRExportPassword,
1453                              &uiSize,
1454                              &uiLog2Contig);
1455     if (eError != PVRSRV_OK)
1456     {
1457                 psExportCookie->uiSize = 0;
1458         goto e0;
1459     }
1460
1461     psExportCookie->hPMRExportHandle = hPMRExportHandle;
1462     psExportCookie->uiPMRExportPassword = uiPMRExportPassword;
1463     psExportCookie->uiSize = uiSize;
1464     psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig;
1465
1466     return PVRSRV_OK;
1467
1468     /*
1469       error exit paths follow
1470     */
1471
1472  e0:
1473     PVR_ASSERT(eError != PVRSRV_OK);
1474     return eError;
1475 }
1476
1477 IMG_INTERNAL PVRSRV_ERROR
1478 DevmemMakeServerExportClientExport(DEVMEM_BRIDGE_HANDLE hBridge,
1479                                    DEVMEM_SERVER_EXPORTCOOKIE hServerExportCookie,
1480                                    DEVMEM_EXPORTCOOKIE *psExportCookie)
1481 {
1482         DEVMEM_EXPORTHANDLE hPMRExportHandle;
1483         IMG_DEVMEM_SIZE_T uiPMRSize;
1484         IMG_DEVMEM_LOG2ALIGN_T uiPMRLog2Contig;
1485         DEVMEM_EXPORTKEY uiPMRExportKey;
1486         PVRSRV_ERROR eError = PVRSRV_OK;
1487
1488         eError = BridgePMRMakeServerExportClientExport(hBridge,
1489                                                                                                         hServerExportCookie,
1490                                                                                                         &hPMRExportHandle,
1491                                                                                                         &uiPMRSize,
1492                                                                                                         &uiPMRLog2Contig,
1493                                                                                                         &uiPMRExportKey);
1494
1495         if (eError == PVRSRV_OK)
1496         {
1497                 psExportCookie->hPMRExportHandle = hPMRExportHandle;
1498                 psExportCookie->uiPMRExportPassword = uiPMRExportKey;
1499                 psExportCookie->uiSize = uiPMRSize;
1500                 psExportCookie->uiLog2ContiguityGuarantee = uiPMRLog2Contig;
1501         }
1502
1503         return eError;
1504 }
1505
1506 IMG_INTERNAL PVRSRV_ERROR
1507 DevmemUnmakeServerExportClientExport(DEVMEM_BRIDGE_HANDLE hBridge,
1508                                    DEVMEM_EXPORTCOOKIE *psExportCookie)
1509 {
1510         return BridgePMRUnmakeServerExportClientExport(hBridge, psExportCookie->hPMRExportHandle);
1511 }
1512
1513 IMG_INTERNAL IMG_BOOL
1514 DevmemIsValidExportCookie(DEVMEM_EXPORTCOOKIE *psExportCookie)
1515 {
1516         /* Assume that if the size is set, the export cookie is used */
1517         return (psExportCookie->uiSize != 0x0);
1518 }
1519
1520 IMG_INTERNAL IMG_VOID
1521 DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
1522                DEVMEM_EXPORTCOOKIE *psExportCookie)
1523 {
1524     _Mapping_Unexport(psMemDesc->psImport,
1525                       psExportCookie->hPMRExportHandle);
1526
1527     psExportCookie->uiSize = 0;
1528 }
1529
1530 IMG_INTERNAL PVRSRV_ERROR
1531 DevmemImport(IMG_HANDLE hBridge,
1532                          DEVMEM_EXPORTCOOKIE *psCookie,
1533                          DEVMEM_FLAGS_T uiFlags,
1534                          DEVMEM_MEMDESC **ppsMemDescPtr)
1535 {
1536     DEVMEM_MEMDESC *psMemDesc = IMG_NULL;
1537     DEVMEM_IMPORT *psImport;
1538     IMG_HANDLE hPMR;
1539     PVRSRV_ERROR eError;
1540
1541         if (ppsMemDescPtr == IMG_NULL)
1542     {
1543         eError = PVRSRV_ERROR_INVALID_PARAMS;
1544         goto failParams;
1545     }
1546
1547         eError =_DevmemMemDescAlloc(&psMemDesc);
1548     if (eError != PVRSRV_OK)
1549     {
1550         goto failMemDescAlloc;
1551     }
1552
1553         eError = _DevmemImportStructAlloc(hBridge,
1554                                                                           IMG_TRUE,
1555                                                                           &psImport);
1556     if (eError != PVRSRV_OK)
1557     {
1558         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1559         goto failImportAlloc;
1560     }
1561
1562
1563     /* Get a handle to the PMR (inc refcount, checks authorization) */
1564     eError = BridgePMRImportPMR(hBridge,
1565                                 psCookie->hPMRExportHandle,
1566                                 psCookie->uiPMRExportPassword,
1567                                 psCookie->uiSize, /* not trusted - just for sanity checks */
1568                                 psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for sanity checks */
1569                                 &hPMR);
1570     if (eError != PVRSRV_OK)
1571     {
1572         goto failImport;
1573     }
1574
1575         _DevmemImportStructInit(psImport,
1576                                                         psCookie->uiSize,
1577                                                         1ULL << psCookie->uiLog2ContiguityGuarantee,
1578                                                         uiFlags,
1579                                                         hPMR);
1580
1581         _DevmemMemDescInit(psMemDesc,
1582                                            0,
1583                                            psImport);
1584
1585     *ppsMemDescPtr = psMemDesc;
1586
1587 #if defined(PVR_RI_DEBUG)
1588         {
1589                 /* Attach RI information */
1590                 eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hBridge,
1591                                                                                         psMemDesc->psImport->hPMR,
1592                                                                                         sizeof("^"),
1593                                                                                         "^",
1594                                                                                         psMemDesc->uiOffset,
1595                                                                                         psMemDesc->psImport->uiSize,
1596                                                                                         IMG_TRUE,
1597                                                                                         IMG_FALSE,
1598                                                                                         &psMemDesc->hRIHandle);
1599                 if( eError != PVRSRV_OK)
1600                 {
1601                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
1602                 }
1603         }
1604 #endif /* if defined(PVR_RI_DEBUG) */
1605
1606         return PVRSRV_OK;
1607
1608     /*
1609       error exit paths follow
1610     */
1611
1612 failImport:
1613     _DevmemImportDiscard(psImport);
1614 failImportAlloc:
1615     _DevmemMemDescDiscard(psMemDesc);
1616 failMemDescAlloc:
1617 failParams:
1618     PVR_ASSERT(eError != PVRSRV_OK);
1619
1620     return eError;
1621 }
1622
1623 /*****************************************************************************
1624  *                   Common MemDesc functions                                *
1625  *****************************************************************************/
1626
1627 /*
1628         This function is called for freeing any class of memory
1629 */
1630 IMG_INTERNAL IMG_VOID
1631 DevmemFree(DEVMEM_MEMDESC *psMemDesc)
1632 {
1633 #if defined(PVR_RI_DEBUG)
1634         if (psMemDesc->hRIHandle)
1635         {
1636             PVRSRV_ERROR eError;
1637
1638             eError = BridgeRIDeleteMEMDESCEntry(psMemDesc->psImport->hBridge,
1639                                                                    psMemDesc->hRIHandle);
1640                 if( eError != PVRSRV_OK)
1641                 {
1642                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIDeleteMEMDESCEntry failed (eError=%d)", __func__, eError));
1643                 }
1644         }
1645 #endif  /* if defined(PVR_RI_DEBUG) */
1646         _DevmemMemDescRelease(psMemDesc);
1647 }
1648
1649 IMG_INTERNAL PVRSRV_ERROR
1650 DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
1651                                   DEVMEM_HEAP *psHeap,
1652                                   IMG_DEV_VIRTADDR *psDevVirtAddr)
1653 {
1654         DEVMEM_IMPORT *psImport;
1655         IMG_DEV_VIRTADDR sDevVAddr;
1656         PVRSRV_ERROR eError;
1657         IMG_BOOL bMap = IMG_TRUE;
1658
1659         OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
1660         if (psHeap == IMG_NULL)
1661         {
1662                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1663                 goto failParams;
1664         }
1665
1666         if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
1667         {
1668                 eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
1669                 goto failCheck;
1670         }
1671
1672         /* Don't map memory for deferred allocations */
1673         if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC)
1674         {
1675                 PVR_ASSERT(psMemDesc->psImport->bExportable);
1676                 bMap = IMG_FALSE;
1677         }
1678
1679         DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
1680                                         __FUNCTION__,
1681                                         psMemDesc,
1682                                         psMemDesc->sDeviceMemDesc.ui32RefCount,
1683                                         psMemDesc->sDeviceMemDesc.ui32RefCount+1);
1684
1685         psImport = psMemDesc->psImport;
1686         _DevmemMemDescAcquire(psMemDesc);
1687
1688         eError = _DevmemImportStructDevMap(psHeap,
1689                                                                            bMap,
1690                                                                            psImport);
1691         if (eError != PVRSRV_OK)
1692         {
1693                 goto failMap;
1694         }
1695
1696         sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
1697         sDevVAddr.uiAddr += psMemDesc->uiOffset;
1698         psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
1699         psMemDesc->sDeviceMemDesc.ui32RefCount++;
1700
1701     *psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
1702
1703     OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1704
1705 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
1706         BridgeDevicememHistoryMap(psMemDesc->psImport->hBridge,
1707                                                 psMemDesc->sDeviceMemDesc.sDevVAddr,
1708                                                 psMemDesc->sTraceData.uiSize,
1709                                                 psMemDesc->sTraceData.szText);
1710 #endif
1711
1712 #if defined(PVR_RI_DEBUG)
1713         if (psMemDesc->hRIHandle)
1714     {
1715                  eError = BridgeRIUpdateMEMDESCAddr(psImport->hBridge,
1716                                                                            psMemDesc->hRIHandle,
1717                                                                            psImport->sDeviceImport.sDevVAddr);
1718                 if( eError != PVRSRV_OK)
1719                 {
1720                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCAddr failed (eError=%d)", __func__, eError));
1721                 }
1722         }
1723 #endif
1724
1725     return PVRSRV_OK;
1726
1727 failMap:
1728         _DevmemMemDescRelease(psMemDesc);
1729 failCheck:
1730 failParams:
1731         OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1732         PVR_ASSERT(eError != PVRSRV_OK);
1733
1734         return eError;
1735 }
1736
1737 IMG_INTERNAL PVRSRV_ERROR
1738 DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
1739                          IMG_DEV_VIRTADDR *psDevVirtAddr)
1740 {
1741         PVRSRV_ERROR eError;
1742
1743         OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
1744         DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
1745                                         __FUNCTION__,
1746                                         psMemDesc,
1747                                         psMemDesc->sDeviceMemDesc.ui32RefCount,
1748                                         psMemDesc->sDeviceMemDesc.ui32RefCount+1);
1749
1750         if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
1751         {
1752                 eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
1753                 goto failCheck;
1754         }
1755         psMemDesc->sDeviceMemDesc.ui32RefCount++;
1756
1757     *psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
1758         OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1759
1760     return PVRSRV_OK;
1761
1762 failCheck:
1763         OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1764         PVR_ASSERT(eError != PVRSRV_OK);
1765
1766         return eError;
1767 }
1768
1769 IMG_INTERNAL IMG_VOID
1770 DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
1771 {
1772         PVR_ASSERT(psMemDesc != IMG_NULL);
1773
1774         OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
1775         DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
1776                                         __FUNCTION__,
1777                                         psMemDesc,
1778                                         psMemDesc->sDeviceMemDesc.ui32RefCount,
1779                                         psMemDesc->sDeviceMemDesc.ui32RefCount-1);
1780
1781         PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0);
1782
1783         if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
1784         {
1785 #if defined(SUPPORT_PAGE_FAULT_DEBUG)
1786                 BridgeDevicememHistoryUnmap(psMemDesc->psImport->hBridge,
1787                                                         psMemDesc->sDeviceMemDesc.sDevVAddr,
1788                                                         psMemDesc->sTraceData.uiSize,
1789                                                         psMemDesc->sTraceData.szText);
1790 #endif
1791                 _DevmemImportStructDevUnmap(psMemDesc->psImport);
1792                 OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1793
1794                 _DevmemMemDescRelease(psMemDesc);
1795         }
1796         else
1797         {
1798                 OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
1799         }
1800 }
1801
1802 IMG_INTERNAL PVRSRV_ERROR
1803 DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
1804                          IMG_VOID **ppvCpuVirtAddr)
1805 {
1806         PVRSRV_ERROR eError;
1807
1808         OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
1809         DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
1810                                         __FUNCTION__,
1811                                         psMemDesc,
1812                                         psMemDesc->sCPUMemDesc.ui32RefCount,
1813                                         psMemDesc->sCPUMemDesc.ui32RefCount+1);
1814
1815         if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0)
1816         {
1817                 DEVMEM_IMPORT *psImport = psMemDesc->psImport;
1818                 IMG_UINT8 *pui8CPUVAddr;
1819
1820                 _DevmemMemDescAcquire(psMemDesc);
1821                 eError = _DevmemImportStructCPUMap(psImport);
1822                 if (eError != PVRSRV_OK)
1823                 {
1824                         goto failMap;
1825                 }
1826
1827                 pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr;
1828                 pui8CPUVAddr += psMemDesc->uiOffset;
1829                 psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr;
1830         }
1831     *ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
1832
1833     VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
1834
1835     OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
1836
1837     return PVRSRV_OK;
1838
1839 failMap:
1840         PVR_ASSERT(eError != PVRSRV_OK);
1841         psMemDesc->sCPUMemDesc.ui32RefCount--;
1842         _DevmemMemDescRelease(psMemDesc);
1843         OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
1844
1845         return eError;
1846 }
1847
1848 IMG_INTERNAL IMG_VOID
1849 DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc)
1850 {
1851         PVR_ASSERT(psMemDesc != IMG_NULL);
1852
1853         OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
1854         DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
1855                                         __FUNCTION__,
1856                                         psMemDesc,
1857                                         psMemDesc->sCPUMemDesc.ui32RefCount,
1858                                         psMemDesc->sCPUMemDesc.ui32RefCount-1);
1859
1860         PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0);
1861
1862         if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0)
1863         {
1864                 OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
1865                 _DevmemImportStructCPUUnmap(psMemDesc->psImport);
1866                 _DevmemMemDescRelease(psMemDesc);
1867         }
1868         else
1869         {
1870                 OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
1871         }
1872 }
1873
1874 IMG_INTERNAL PVRSRV_ERROR
1875 DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
1876                            IMG_HANDLE *phImport)
1877 {
1878         if (psMemDesc->psImport->bExportable == IMG_FALSE)
1879         {
1880                 return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
1881         }
1882
1883         *phImport = psMemDesc->psImport->hPMR;
1884
1885         return PVRSRV_OK;
1886 }
1887
1888 IMG_INTERNAL PVRSRV_ERROR
1889 DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
1890                                                    IMG_UINT64 *pui64UID)
1891 {
1892         DEVMEM_IMPORT *psImport = psMemDesc->psImport;
1893         PVRSRV_ERROR eError;
1894
1895         eError = BridgePMRGetUID(psImport->hBridge,
1896                                                          psImport->hPMR,
1897                                                          pui64UID);
1898
1899         return eError;
1900 }
1901
1902 IMG_INTERNAL PVRSRV_ERROR
1903 DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
1904                                 IMG_HANDLE *hReservation)
1905 {
1906         DEVMEM_IMPORT *psImport;
1907
1908         PVR_ASSERT(psMemDesc);
1909         psImport = psMemDesc->psImport;
1910
1911         PVR_ASSERT(psImport);
1912         *hReservation = psImport->sDeviceImport.hReservation;
1913
1914         return PVRSRV_OK;
1915 }
1916
1917 PVRSRV_ERROR
1918 DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
1919                 IMG_HANDLE *phPMR,
1920                 IMG_DEVMEM_OFFSET_T *puiPMROffset)
1921 {
1922         DEVMEM_IMPORT *psImport;
1923
1924         PVR_ASSERT(psMemDesc);
1925         *puiPMROffset = psMemDesc->uiOffset;
1926         psImport = psMemDesc->psImport;
1927
1928         PVR_ASSERT(psImport);
1929         *phPMR = psImport->hPMR;
1930
1931         return PVRSRV_OK;
1932 }
1933
1934 IMG_INTERNAL PVRSRV_ERROR
1935 DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
1936                                 DEVMEM_FLAGS_T *puiFlags)
1937 {
1938         DEVMEM_IMPORT *psImport;
1939
1940         PVR_ASSERT(psMemDesc);
1941         psImport = psMemDesc->psImport;
1942
1943         PVR_ASSERT(psImport);
1944         *puiFlags = psImport->uiFlags;
1945
1946         return PVRSRV_OK;
1947 }
1948
1949 IMG_INTERNAL PVRSRV_ERROR
1950 DevmemLocalImport(IMG_HANDLE hBridge,
1951                                   IMG_HANDLE hExtHandle,
1952                                   DEVMEM_FLAGS_T uiFlags,
1953                                   DEVMEM_MEMDESC **ppsMemDescPtr,
1954                                   IMG_DEVMEM_SIZE_T *puiSizePtr)
1955 {
1956     DEVMEM_MEMDESC *psMemDesc = IMG_NULL;
1957     DEVMEM_IMPORT *psImport;
1958     IMG_DEVMEM_SIZE_T uiSize;
1959     IMG_DEVMEM_ALIGN_T uiAlign;
1960     IMG_HANDLE hPMR;
1961     PVRSRV_ERROR eError;
1962
1963     if (ppsMemDescPtr == IMG_NULL)
1964     {
1965         eError = PVRSRV_ERROR_INVALID_PARAMS;
1966         goto failParams;
1967     }
1968
1969         eError =_DevmemMemDescAlloc(&psMemDesc);
1970     if (eError != PVRSRV_OK)
1971     {
1972         goto failMemDescAlloc;
1973     }
1974
1975         eError = _DevmemImportStructAlloc(hBridge,
1976                                                                           IMG_TRUE,
1977                                                                           &psImport);
1978     if (eError != PVRSRV_OK)
1979     {
1980         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
1981         goto failImportAlloc;
1982     }
1983
1984         /* Get the PMR handle and it's size from the server */
1985         eError = BridgePMRLocalImportPMR(hBridge,
1986                                                                          hExtHandle,
1987                                                                          &hPMR,
1988                                                                          &uiSize,
1989                                                                          &uiAlign);
1990         if (eError != PVRSRV_OK)
1991         {
1992                 goto failImport;
1993         }
1994
1995         _DevmemImportStructInit(psImport,
1996                                                         uiSize,
1997                                                         uiAlign,
1998                                                         uiFlags,
1999                                                         hPMR);
2000
2001         _DevmemMemDescInit(psMemDesc,
2002                                            0,
2003                                            psImport);
2004
2005     *ppsMemDescPtr = psMemDesc;
2006         if (puiSizePtr)
2007                 *puiSizePtr = uiSize;
2008
2009 #if defined(PVR_RI_DEBUG)
2010         {
2011                 /* Attach RI information */
2012                 eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hBridge,
2013                                                                                         psMemDesc->psImport->hPMR,
2014                                                                                         sizeof("^"),
2015                                                                                         "^",
2016                                                                                         psMemDesc->uiOffset,
2017                                                                                         psMemDesc->psImport->uiSize,
2018                                                                                         IMG_TRUE,
2019                                                                                         IMG_FALSE,
2020                                                                                         &(psMemDesc->hRIHandle));
2021                 if( eError != PVRSRV_OK)
2022                 {
2023                         PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
2024                 }
2025         }
2026 #endif /* if defined(PVR_RI_DEBUG) */
2027         return PVRSRV_OK;
2028
2029 failImport:
2030     _DevmemImportDiscard(psImport);
2031 failImportAlloc:
2032         _DevmemMemDescDiscard(psMemDesc);
2033 failMemDescAlloc:
2034 failParams:
2035         PVR_ASSERT(eError != PVRSRV_OK);
2036
2037         return eError;
2038 }
2039
2040 IMG_INTERNAL PVRSRV_ERROR
2041 DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
2042                          IMG_DEV_VIRTADDR sDevVAddr)
2043 {
2044     return BridgeDevmemIsVDevAddrValid(psContext->hBridge,
2045                                        psContext->hDevMemServerContext,
2046                                        sDevVAddr);
2047 }
2048
2049 IMG_INTERNAL IMG_UINT32
2050 DevmemGetHeapLog2ImportAlignment(DEVMEM_HEAP *psHeap)
2051 {
2052         return psHeap->uiLog2ImportAlignment;
2053 }
2054