drivers/gpu/rogue/services/server/env/linux/physmem_dmabuf.c

   1 /*************************************************************************/ /*!
   2 @File           physmem_dmabuf.c
   3 @Title          dmabuf memory allocator
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Part of the memory management. This module is responsible for
   6                 implementing the function callbacks for dmabuf memory.
   7 @License        Dual MIT/GPLv2
   8
   9 The contents of this file are subject to the MIT license as set out below.
  10
  11 Permission is hereby granted, free of charge, to any person obtaining a copy
  12 of this software and associated documentation files (the "Software"), to deal
  13 in the Software without restriction, including without limitation the rights
  14 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  15 copies of the Software, and to permit persons to whom the Software is
  16 furnished to do so, subject to the following conditions:
  17
  18 The above copyright notice and this permission notice shall be included in
  19 all copies or substantial portions of the Software.
  20
  21 Alternatively, the contents of this file may be used under the terms of
  22 the GNU General Public License Version 2 ("GPL") in which case the provisions
  23 of GPL are applicable instead of those above.
  24
  25 If you wish to allow use of your version of this file only under the terms of
  26 GPL, and not to allow others to use your version of this file under the terms
  27 of the MIT license, indicate your decision by deleting the provisions above
  28 and replace them with the notice and other provisions required by GPL as set
  29 out in the file called "GPL-COPYING" included in this distribution. If you do
  30 not delete the provisions above, a recipient may use your version of this file
  31 under the terms of either the MIT license or GPL.
  32
  33 This License is also included in this distribution in the file called
  34 "MIT-COPYING".
  35
  36 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  37 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  38 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  39 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  40 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  41 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  42 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  43 */ /**************************************************************************/
  44
  45 #include "img_types.h"
  46 #include "pvr_debug.h"
  47 #include "pvrsrv_error.h"
  48 #include "pvrsrv_memallocflags.h"
  49
  50 #include "allocmem.h"
  51 #include "osfunc.h"
  52 #include "pvrsrv.h"
  53 #include "physmem_lma.h"
  54 #include "pdump_physmem.h"
  55 #include "pmr.h"
  56 #include "pmr_impl.h"
  57 #include "physmem_dmabuf.h"
  58 #include "ion_sys.h"
  59 #include "hash.h"
  60
  61 #if defined(PVR_RI_DEBUG)
  62 #include "ri_server.h"
  63 #endif
  64
  65 #include <linux/err.h>
  66 #include <linux/slab.h>
  67 #include <linux/dma-buf.h>
  68 #include <linux/scatterlist.h>
  69
  70 #if defined(LDM_PCI)
  71 #include <linux/pci.h>
  72 #elif defined(LDM_PLATFORM)
  73 #include <linux/platform_device.h>
  74 #else
  75 #error Either LDM_PCI or LDM_PLATFORM must be defined
  76 #endif
  77
  78 typedef struct _PMR_DMA_BUF_DATA_
  79 {
  80         struct dma_buf_attachment *psAttachment;
  81         struct sg_table *psSgTable;
  82         struct dma_buf *psDmaBuf;
  83
  84         IMG_DEVMEM_SIZE_T uiSize;
  85         IMG_PVOID *pvKernAddr;
  86
  87         IMG_DEV_PHYADDR *pasDevPhysAddr;
  88         IMG_UINT32 ui32PageCount;
  89         PHYS_HEAP *psPhysHeap;
  90
  91         IMG_BOOL bPoisonOnFree;
  92         IMG_BOOL bPDumpMalloced;
  93         IMG_HANDLE hPDumpAllocInfo;
  94 } PMR_DMA_BUF_DATA;
  95
  96 /* Start size of the g_psDmaBufHash hash table */
  97 #define DMA_BUF_HASH_SIZE 20
  98
  99 extern
 100 #if defined(LDM_PCI)
 101         struct pci_dev
 102 #elif defined(LDM_PLATFORM)
 103         struct platform_device
 104 #endif
 105         *gpsPVRLDMDev;
 106
 107 static HASH_TABLE *g_psDmaBufHash = IMG_NULL;
 108 static IMG_UINT32 g_ui32HashRefCount = 0;
 109
 110 #if defined(PVR_ANDROID_ION_USE_SG_LENGTH)
 111 #define pvr_sg_length(sg) ((sg)->length)
 112 #else
 113 #define pvr_sg_length(sg) sg_dma_len(sg)
 114 #endif
 115
 116 /*****************************************************************************
 117  *                     DMA-BUF specific functions                            *
 118  *****************************************************************************/
 119
 120 /*
 121         Obtain a list of physical pages from the dmabuf.
 122 */
 123 static PVRSRV_ERROR DmaBufPhysAddrAcquire(PMR_DMA_BUF_DATA *psPrivData, int fd)
 124 {
 125         struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
 126         IMG_DEV_PHYADDR *pasDevPhysAddr = NULL;
 127         IMG_CPU_PHYADDR sCpuPhysAddr;
 128         IMG_UINT32 ui32PageCount = 0;
 129         struct scatterlist *sg;
 130         struct sg_table *table;
 131         PVRSRV_ERROR eError;
 132         IMG_UINT32 i;
 133
 134         table = dma_buf_map_attachment(psAttachment, DMA_NONE);
 135         if (!table)
 136         {
 137                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 138                 goto exitFailMap;
 139         }
 140
 141         /*
 142                 We do a two pass process, 1st workout how many pages there
 143                 are, 2nd fill in the data.
 144         */
 145         for_each_sg(table->sgl, sg, table->nents, i)
 146         {
 147                 ui32PageCount += PAGE_ALIGN(pvr_sg_length(sg)) / PAGE_SIZE;
 148         }
 149
 150         if (WARN_ON(!ui32PageCount))
 151         {
 152                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to import dmabuf with no pages",
 153                                  __func__));
 154                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 155                 goto exitFailMap;
 156         }
 157
 158         pasDevPhysAddr = OSAllocMem(sizeof(IMG_DEV_PHYADDR)*ui32PageCount);
 159         if (!pasDevPhysAddr)
 160         {
 161                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 162                 goto exitFailAlloc;
 163         }
 164
 165         ui32PageCount = 0;
 166
 167         for_each_sg(table->sgl, sg, table->nents, i)
 168         {
 169                 IMG_UINT32 j;
 170
 171                 for (j = 0; j < pvr_sg_length(sg); j += PAGE_SIZE)
 172                 {
 173                         /* Pass 2: Get the page data */
 174                         sCpuPhysAddr.uiAddr = sg_phys(sg);
 175
 176                         pasDevPhysAddr[ui32PageCount] =
 177                                 IonCPUPhysToDevPhys(sCpuPhysAddr, j);
 178                         ui32PageCount++;
 179                 }
 180         }
 181
 182         psPrivData->pasDevPhysAddr = pasDevPhysAddr;
 183         psPrivData->ui32PageCount = ui32PageCount;
 184         psPrivData->uiSize = (IMG_DEVMEM_SIZE_T)ui32PageCount * PAGE_SIZE;
 185         psPrivData->psSgTable = table;
 186
 187         return PVRSRV_OK;
 188
 189 exitFailAlloc:
 190 exitFailMap:
 191         PVR_ASSERT(eError!= PVRSRV_OK);
 192         return eError;
 193 }
 194
 195 static void DmaBufPhysAddrRelease(PMR_DMA_BUF_DATA *psPrivData)
 196 {
 197         struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
 198         struct sg_table *psSgTable = psPrivData->psSgTable;
 199
 200         dma_buf_unmap_attachment(psAttachment, psSgTable, DMA_NONE);
 201
 202         OSFreeMem(psPrivData->pasDevPhysAddr);
 203 }
 204
 205 static IMG_BOOL _DmaBufKeyCompare(IMG_SIZE_T uKeySize, void *pKey1, void *pKey2)
 206 {
 207         IMG_DEV_PHYADDR *psKey1 = pKey1;
 208         IMG_DEV_PHYADDR *psKey2 = pKey2;
 209         PVR_ASSERT(uKeySize == sizeof(IMG_DEV_PHYADDR));
 210
 211         return psKey1->uiAddr == psKey2->uiAddr;
 212 }
 213
 214 /*****************************************************************************
 215  *                       PMR callback functions                              *
 216  *****************************************************************************/
 217
 218
 219 static void _Poison(IMG_PVOID pvKernAddr, IMG_DEVMEM_SIZE_T uiBufferSize,
 220                                         const IMG_CHAR *pacPoisonData, IMG_SIZE_T uiPoisonSize)
 221 {
 222         IMG_DEVMEM_SIZE_T uiDestByteIndex;
 223         IMG_CHAR *pcDest = pvKernAddr;
 224         IMG_UINT32 uiSrcByteIndex = 0;
 225
 226         for(uiDestByteIndex=0; uiDestByteIndex<uiBufferSize; uiDestByteIndex++)
 227         {
 228                 pcDest[uiDestByteIndex] = pacPoisonData[uiSrcByteIndex];
 229                 uiSrcByteIndex++;
 230                 if (uiSrcByteIndex == uiPoisonSize)
 231                 {
 232                         uiSrcByteIndex = 0;
 233                 }
 234         }
 235 }
 236
 237 static const IMG_CHAR _AllocPoison[] = "^PoIsOn";
 238 static const IMG_UINT32 _AllocPoisonSize = 7;
 239 static const IMG_CHAR _FreePoison[] = "<DEAD-BEEF>";
 240 static const IMG_UINT32 _FreePoisonSize = 11;
 241
 242 static PVRSRV_ERROR PMRFinalizeDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
 243 {
 244         PMR_DMA_BUF_DATA *psPrivData = pvPriv;
 245
 246         HASH_Remove_Extended(g_psDmaBufHash, &psPrivData->pasDevPhysAddr[0]);
 247         g_ui32HashRefCount--;
 248         if (g_ui32HashRefCount == 0)
 249         {
 250                 HASH_Delete(g_psDmaBufHash);
 251                 g_psDmaBufHash = IMG_NULL;
 252         }
 253
 254         if (psPrivData->bPDumpMalloced)
 255         {
 256                 PDumpPMRFree(psPrivData->hPDumpAllocInfo);
 257         }
 258
 259         if (psPrivData->bPoisonOnFree)
 260         {
 261                 IMG_PVOID pvKernAddr;
 262                 int i, err;
 263
 264                 err = dma_buf_begin_cpu_access(psPrivData->psDmaBuf, 0,
 265                                                                            psPrivData->uiSize, DMA_NONE);
 266                 if (err)
 267                 {
 268                         PVR_DPF((PVR_DBG_ERROR, "%s: Failed to begin cpu access", __func__));
 269                         PVR_ASSERT(IMG_FALSE);
 270                 }
 271
 272                 for (i = 0; i < psPrivData->uiSize / PAGE_SIZE; i++)
 273                 {
 274                         pvKernAddr = dma_buf_kmap(psPrivData->psDmaBuf, i);
 275
 276                         if (IS_ERR_OR_NULL(pvKernAddr))
 277                         {
 278                                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to poison allocation before free", __func__));
 279                                 PVR_ASSERT(IMG_FALSE);
 280                         }
 281
 282                         _Poison(pvKernAddr, PAGE_SIZE, _FreePoison, _FreePoisonSize);
 283
 284                         dma_buf_kunmap(psPrivData->psDmaBuf, i, pvKernAddr);
 285                 }
 286
 287                 dma_buf_end_cpu_access(psPrivData->psDmaBuf, 0,
 288                                                            psPrivData->uiSize, DMA_NONE);
 289         }
 290
 291         DmaBufPhysAddrRelease(psPrivData);
 292         dma_buf_detach(psPrivData->psDmaBuf, psPrivData->psAttachment);
 293         dma_buf_put(psPrivData->psDmaBuf);
 294         PhysHeapRelease(psPrivData->psPhysHeap);
 295         OSFreeMem(psPrivData);
 296
 297         return PVRSRV_OK;
 298 }
 299
 300 /*
 301         Lock and unlock function for physical address
 302         don't do anything for as we acquire the physical
 303         address at create time.
 304 */
 305 static PVRSRV_ERROR PMRLockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
 306                                                                                            IMG_UINT32 uiLog2DevPageSize)
 307 {
 308         PVR_UNREFERENCED_PARAMETER(pvPriv);
 309         PVR_UNREFERENCED_PARAMETER(uiLog2DevPageSize);
 310
 311         return PVRSRV_OK;
 312
 313 }
 314
 315 static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
 316 {
 317         PVR_UNREFERENCED_PARAMETER(pvPriv);
 318
 319         return PVRSRV_OK;
 320 }
 321
 322 static PVRSRV_ERROR PMRDevPhysAddrDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
 323                                                                                  IMG_DEVMEM_OFFSET_T uiOffset,
 324                                                                                  IMG_DEV_PHYADDR *psDevPAddr)
 325 {
 326         PMR_DMA_BUF_DATA *psPrivData = pvPriv;
 327         IMG_UINT32 ui32PageCount;
 328         IMG_UINT32 ui32PageIndex;
 329         IMG_UINT32 ui32InPageOffset;
 330
 331         ui32PageCount = psPrivData->ui32PageCount;
 332
 333         ui32PageIndex = uiOffset >> PAGE_SHIFT;
 334         ui32InPageOffset = uiOffset - ((IMG_DEVMEM_OFFSET_T)ui32PageIndex << PAGE_SHIFT);
 335         PVR_ASSERT(ui32PageIndex < ui32PageCount);
 336         PVR_ASSERT(ui32InPageOffset < PAGE_SIZE);
 337
 338         psDevPAddr->uiAddr = psPrivData->pasDevPhysAddr[ui32PageIndex].uiAddr + ui32InPageOffset;
 339
 340         return PVRSRV_OK;
 341 }
 342
 343 static PVRSRV_ERROR
 344 PMRAcquireKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
 345                                                                   IMG_SIZE_T uiOffset,
 346                                                                   IMG_SIZE_T uiSize,
 347                                                                   void **ppvKernelAddressOut,
 348                                                                   IMG_HANDLE *phHandleOut,
 349                                                                   PMR_FLAGS_T ulFlags)
 350 {
 351         PMR_DMA_BUF_DATA *psPrivData = pvPriv;
 352         IMG_PVOID pvKernAddr;
 353         PVRSRV_ERROR eError;
 354         int err;
 355
 356         PVR_ASSERT(psPrivData->pvKernAddr == IMG_NULL);
 357
 358         err = dma_buf_begin_cpu_access(psPrivData->psDmaBuf, 0,
 359                                                                    psPrivData->uiSize, DMA_NONE);
 360         if (err)
 361         {
 362                 eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
 363                 goto fail;
 364         }
 365
 366         pvKernAddr = dma_buf_vmap(psPrivData->psDmaBuf);
 367         if (IS_ERR_OR_NULL(pvKernAddr))
 368         {
 369                 eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
 370                 goto fail_kmap;
 371         }
 372
 373         *ppvKernelAddressOut = pvKernAddr + uiOffset;
 374         psPrivData->pvKernAddr = pvKernAddr;
 375         return PVRSRV_OK;
 376
 377 fail_kmap:
 378         dma_buf_end_cpu_access(psPrivData->psDmaBuf, 0,
 379                                                    psPrivData->uiSize, DMA_NONE);
 380 fail:
 381         PVR_ASSERT(eError != PVRSRV_OK);
 382         return eError;
 383 }
 384
 385 static void PMRReleaseKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
 386                                                                                           IMG_HANDLE hHandle)
 387 {
 388         PMR_DMA_BUF_DATA *psPrivData = pvPriv;
 389
 390         PVR_UNREFERENCED_PARAMETER(hHandle);
 391
 392         dma_buf_vunmap(psPrivData->psDmaBuf, psPrivData->pvKernAddr);
 393         psPrivData->pvKernAddr = IMG_NULL;
 394
 395         dma_buf_end_cpu_access(psPrivData->psDmaBuf, 0,
 396                                                    psPrivData->uiSize, DMA_NONE);
 397 }
 398
 399 static PMR_IMPL_FUNCTAB _sPMRDmaBufFuncTab =
 400 {
 401         .pfnLockPhysAddresses                   = PMRLockPhysAddressesDmaBuf,
 402         .pfnUnlockPhysAddresses                 = PMRUnlockPhysAddressesDmaBuf,
 403         .pfnDevPhysAddr                                 = PMRDevPhysAddrDmaBuf,
 404         .pfnAcquireKernelMappingData    = PMRAcquireKernelMappingDataDmaBuf,
 405         .pfnReleaseKernelMappingData    = PMRReleaseKernelMappingDataDmaBuf,
 406         .pfnFinalize                                    = PMRFinalizeDmaBuf,
 407 };
 408
 409 /*****************************************************************************
 410  *                       Public facing interface                             *
 411  *****************************************************************************/
 412
 413 PVRSRV_ERROR
 414 PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
 415                                         IMG_INT fd,
 416                                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
 417                                         PMR **ppsPMRPtr,
 418                                         IMG_DEVMEM_SIZE_T *puiSize,
 419                                         IMG_DEVMEM_ALIGN_T *puiAlign)
 420 {
 421         PMR_DMA_BUF_DATA *psPrivData = IMG_NULL;
 422         IMG_HANDLE hPDumpAllocInfo = IMG_NULL;
 423         IMG_BOOL bMappingTable = IMG_TRUE;
 424         IMG_BOOL bPoisonOnAlloc;
 425         IMG_BOOL bPoisonOnFree;
 426         PMR_FLAGS_T uiPMRFlags;
 427         PMR *psPMR = IMG_NULL;
 428         PVRSRV_ERROR eError;
 429         IMG_BOOL bZero;
 430
 431         if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
 432         {
 433                 bZero = IMG_TRUE;
 434         }
 435         else
 436         {
 437                 bZero = IMG_FALSE;
 438         }
 439
 440         if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC)
 441         {
 442                 bPoisonOnAlloc = IMG_TRUE;
 443         }
 444         else
 445         {
 446                 bPoisonOnAlloc = IMG_FALSE;
 447         }
 448
 449         if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_FREE)
 450         {
 451                 bPoisonOnFree = IMG_TRUE;
 452         }
 453         else
 454         {
 455                 bPoisonOnFree = IMG_FALSE;
 456         }
 457
 458         if ((uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &&
 459                 (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
 460         {
 461                 /* Zero on Alloc and Poison on Alloc are mutually exclusive */
 462                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 463                 goto fail_params;
 464         }
 465
 466         if (!psConnection)
 467         {
 468                 eError = PVRSRV_ERROR_INVALID_PARAMS;
 469                 goto fail_params;
 470         }
 471
 472         psPrivData = OSAllocMem(sizeof(*psPrivData));
 473         if (psPrivData == NULL)
 474         {
 475                 eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 476                 goto fail_privalloc;
 477         }
 478
 479         /*
 480                 Get the physical heap for this PMR
 481
 482                 Note:
 483                 While we have no way to determine the type of the buffer
 484                 we just assume that all dmabufs are from the same
 485                 physical heap.
 486         */
 487         eError = PhysHeapAcquire(IonPhysHeapID(), &psPrivData->psPhysHeap);
 488         if (eError != PVRSRV_OK)
 489         {
 490                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed PhysHeapAcquire", __func__));
 491                 goto fail_physheap;
 492         }
 493
 494         /* Get the buffer handle */
 495         psPrivData->psDmaBuf = dma_buf_get(fd);
 496
 497         if (IS_ERR_OR_NULL(psPrivData->psDmaBuf))
 498         {
 499                 PVR_DPF((PVR_DBG_ERROR, "%s: dma_buf_get failed", __func__));
 500                 eError = PVRSRV_ERROR_BAD_MAPPING;
 501                 goto fail_dma_buf_get;
 502         }
 503
 504         /* Attach a fake device to to the dmabuf */
 505         psPrivData->psAttachment = dma_buf_attach(psPrivData->psDmaBuf, &gpsPVRLDMDev->dev);
 506
 507         if (IS_ERR_OR_NULL(psPrivData->psAttachment))
 508         {
 509                 PVR_DPF((PVR_DBG_ERROR, "%s: dma_buf_get failed", __func__));
 510                 eError = PVRSRV_ERROR_BAD_MAPPING;
 511                 goto fail_attach;
 512         }
 513
 514         /*
 515                 Note:
 516
 517                 We could defer the import until lock address time but we
 518                 do it here as then we can detect any errors at import time.
 519                 Also we need to know the dmabuf size here and there seems
 520                 to be no other way to find that other then map the buffer for dma.
 521         */
 522         eError = DmaBufPhysAddrAcquire(psPrivData, fd);
 523         if (eError != PVRSRV_OK)
 524         {
 525                 PVR_DPF((PVR_DBG_ERROR, "%s: DmaBufPhysAddrAcquire failed", __func__));
 526                 goto fail_acquire;
 527         }
 528
 529         if (g_psDmaBufHash == IMG_NULL)
 530         {
 531                 /*
 532                         As different processes may import the same dmabuf we need to
 533                         create a hash table so we don't generate a duplicate PMR but
 534                         rather just take a reference on an existing one.
 535                 */
 536                 g_psDmaBufHash = HASH_Create_Extended(DMA_BUF_HASH_SIZE, sizeof(psPrivData->pasDevPhysAddr[0]), HASH_Func_Default, _DmaBufKeyCompare);
 537                 if (g_psDmaBufHash == IMG_NULL)
 538                 {
 539                         eError = PVRSRV_ERROR_OUT_OF_MEMORY;
 540                 }
 541         }
 542         else
 543         {
 544                 /*
 545                         We have a hash table so check if have already seen this
 546                         this dmabuf before
 547                 */
 548                 psPMR = (PMR *) HASH_Retrieve_Extended(g_psDmaBufHash, &psPrivData->pasDevPhysAddr[0]);
 549                 if (psPMR != IMG_NULL)
 550                 {
 551                         /*
 552                                 We already know about this dmabuf but we had to do a bunch
 553                                 for work to determine that so here we have to undo it
 554                         */
 555                         DmaBufPhysAddrRelease(psPrivData);
 556                         dma_buf_detach(psPrivData->psDmaBuf, psPrivData->psAttachment);
 557                         dma_buf_put(psPrivData->psDmaBuf);
 558                         PhysHeapRelease(psPrivData->psPhysHeap);
 559                         OSFreeMem(psPrivData);
 560
 561                         /* Reuse the PMR we already created */
 562                         PMRRefPMR(psPMR);
 563
 564                         *ppsPMRPtr = psPMR;
 565                         psPrivData = PMRGetPrivateDataHack(psPMR, &_sPMRDmaBufFuncTab);
 566                         *puiSize = psPrivData->uiSize;
 567                         *puiAlign = PAGE_SIZE;
 568                         return PVRSRV_OK;
 569                 }
 570         }
 571
 572         if (bZero || bPoisonOnAlloc)
 573         {
 574                 IMG_PVOID pvKernAddr;
 575                 int i, err;
 576
 577                 err = dma_buf_begin_cpu_access(psPrivData->psDmaBuf, 0,
 578                                                                            psPrivData->uiSize, DMA_NONE);
 579                 if (err)
 580                 {
 581                         eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
 582                         goto fail_begin;
 583                 }
 584
 585                 for (i = 0; i < psPrivData->uiSize / PAGE_SIZE; i++)
 586                 {
 587                         pvKernAddr = dma_buf_kmap(psPrivData->psDmaBuf, i);
 588
 589                         if (IS_ERR_OR_NULL(pvKernAddr))
 590                         {
 591                                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to poison allocation before free", __func__));
 592                                 eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
 593                                 goto fail_kmap;
 594                         }
 595
 596                         if (bZero)
 597                         {
 598                                 memset(pvKernAddr, 0, PAGE_SIZE);
 599                         }
 600                         else
 601                         {
 602                                 _Poison(pvKernAddr, PAGE_SIZE, _AllocPoison, _AllocPoisonSize);
 603                         }
 604
 605                         dma_buf_kunmap(psPrivData->psDmaBuf, i, pvKernAddr);
 606                 }
 607
 608                 dma_buf_end_cpu_access(psPrivData->psDmaBuf, 0,
 609                                                            psPrivData->uiSize, DMA_NONE);
 610         }
 611
 612         psPrivData->bPoisonOnFree = bPoisonOnFree;
 613
 614         uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
 615         /* check no significant bits were lost in cast due to different
 616            bit widths for flags */
 617         PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
 618
 619         eError = PMRCreatePMR(psPrivData->psPhysHeap,
 620                                                   psPrivData->uiSize,
 621                           psPrivData->uiSize,
 622                           1,
 623                           1,
 624                           &bMappingTable,
 625                                                   PAGE_SHIFT,
 626                                                   uiPMRFlags,
 627                                                   "PMRDMABUF",
 628                                                   &_sPMRDmaBufFuncTab,
 629                                                   psPrivData,
 630                                                   &psPMR,
 631                                                   &hPDumpAllocInfo,
 632                                                   IMG_FALSE);
 633         if (eError != PVRSRV_OK)
 634         {
 635                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create PMR", __func__));
 636                 goto fail_pmrcreate;
 637         }
 638
 639 #if defined(PVR_RI_DEBUG)
 640         {
 641                 eError = RIWritePMREntryKM (psPMR,
 642                                                                         sizeof("DMABUF"),
 643                                                                         "DMABUF",
 644                                                                         psPrivData->uiSize);
 645         }
 646 #endif
 647
 648         psPrivData->hPDumpAllocInfo = hPDumpAllocInfo;
 649         psPrivData->bPDumpMalloced = IMG_TRUE;
 650
 651         /* First time we've seen this dmabuf so store it in the hash table */
 652         HASH_Insert_Extended(g_psDmaBufHash, &psPrivData->pasDevPhysAddr[0], (IMG_UINTPTR_T) psPMR);
 653         g_ui32HashRefCount++;
 654
 655         *ppsPMRPtr = psPMR;
 656         *puiSize = psPrivData->uiSize;
 657         *puiAlign = PAGE_SIZE;
 658         return PVRSRV_OK;
 659
 660 fail_pmrcreate:
 661 fail_kmap:
 662         dma_buf_end_cpu_access(psPrivData->psDmaBuf, 0,
 663                                                    psPrivData->uiSize, DMA_NONE);
 664 fail_begin:
 665         DmaBufPhysAddrRelease(psPrivData);
 666 fail_acquire:
 667         dma_buf_detach(psPrivData->psDmaBuf, psPrivData->psAttachment);
 668 fail_attach:
 669         dma_buf_put(psPrivData->psDmaBuf);
 670 fail_dma_buf_get:
 671         PhysHeapRelease(psPrivData->psPhysHeap);
 672 fail_physheap:
 673         OSFreeMem(psPrivData);
 674
 675 fail_privalloc:
 676 fail_params:
 677         PVR_ASSERT(eError != PVRSRV_OK);
 678         return eError;
 679 }