drivers/gpu/arm/midgard/mali_kbase_mem_linux.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_mem_linux.c
  22  * Base kernel memory APIs, Linux implementation.
  23  */
  24
  25 #include <linux/compat.h>
  26 #include <linux/kernel.h>
  27 #include <linux/bug.h>
  28 #include <linux/mm.h>
  29 #include <linux/mman.h>
  30 #include <linux/fs.h>
  31 #include <linux/version.h>
  32 #include <linux/dma-mapping.h>
  33 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
  34         #include <linux/dma-attrs.h>
  35 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)  */
  36 #ifdef CONFIG_DMA_SHARED_BUFFER
  37 #include <linux/dma-buf.h>
  38 #endif                          /* defined(CONFIG_DMA_SHARED_BUFFER) */
  39 #include <linux/shrinker.h>
  40
  41 #include <mali_kbase.h>
  42 #include <mali_kbase_mem_linux.h>
  43 #include <mali_kbase_config_defaults.h>
  44 #include <mali_kbase_hwaccess_time.h>
  45 #include <mali_kbase_tlstream.h>
  46
  47 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
  48 static const struct vm_operations_struct kbase_vm_ops;
  49
  50 /**
  51  * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation
  52  * @kctx:      Context the region belongs to
  53  * @reg:       The GPU region
  54  * @new_pages: The number of pages after the shrink
  55  * @old_pages: The number of pages before the shrink
  56  *
  57  * Return: 0 on success, -errno on error.
  58  *
  59  * Shrink (or completely remove) all CPU mappings which reference the shrunk
  60  * part of the allocation.
  61  *
  62  * Note: Caller must be holding the processes mmap_sem lock.
  63  */
  64 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
  65                 struct kbase_va_region *reg,
  66                 u64 new_pages, u64 old_pages);
  67
  68 /**
  69  * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
  70  * @kctx:      Context the region belongs to
  71  * @reg:       The GPU region or NULL if there isn't one
  72  * @new_pages: The number of pages after the shrink
  73  * @old_pages: The number of pages before the shrink
  74  *
  75  * Return: 0 on success, negative -errno on error
  76  *
  77  * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
  78  * itself is unmodified as we still need to reserve the VA, only the page tables
  79  * will be modified by this function.
  80  */
  81 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
  82                 struct kbase_va_region *reg,
  83                 u64 new_pages, u64 old_pages);
  84
  85 struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, u64 *gpu_va, u16 *va_alignment)
  86 {
  87         int zone;
  88         int gpu_pc_bits;
  89         int cpu_va_bits;
  90         struct kbase_va_region *reg;
  91         struct device *dev;
  92
  93         KBASE_DEBUG_ASSERT(kctx);
  94         KBASE_DEBUG_ASSERT(flags);
  95         KBASE_DEBUG_ASSERT(gpu_va);
  96         KBASE_DEBUG_ASSERT(va_alignment);
  97
  98         dev = kctx->kbdev->dev;
  99         *va_alignment = 0; /* no alignment by default */
 100         *gpu_va = 0; /* return 0 on failure */
 101
 102         gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
 103         cpu_va_bits = BITS_PER_LONG;
 104
 105         if (0 == va_pages) {
 106                 dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!");
 107                 goto bad_size;
 108         }
 109
 110         if (va_pages > (U64_MAX / PAGE_SIZE))
 111                 /* 64-bit address range is the max */
 112                 goto bad_size;
 113
 114 #if defined(CONFIG_64BIT)
 115         if (kctx->is_compat)
 116                 cpu_va_bits = 32;
 117 #endif
 118
 119         if (!kbase_check_alloc_flags(*flags)) {
 120                 dev_warn(dev,
 121                                 "kbase_mem_alloc called with bad flags (%llx)",
 122                                 (unsigned long long)*flags);
 123                 goto bad_flags;
 124         }
 125
 126         if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
 127                         !kbase_device_is_cpu_coherent(kctx->kbdev)) {
 128                 dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
 129                 goto bad_flags;
 130         }
 131         if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
 132                         !kbase_device_is_cpu_coherent(kctx->kbdev)) {
 133                 /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
 134                 *flags &= ~BASE_MEM_COHERENT_SYSTEM;
 135         }
 136
 137         /* Limit GPU executable allocs to GPU PC size */
 138         if ((*flags & BASE_MEM_PROT_GPU_EX) &&
 139             (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT)))
 140                 goto bad_ex_size;
 141
 142         /* find out which VA zone to use */
 143         if (*flags & BASE_MEM_SAME_VA)
 144                 zone = KBASE_REG_ZONE_SAME_VA;
 145         else if (*flags & BASE_MEM_PROT_GPU_EX)
 146                 zone = KBASE_REG_ZONE_EXEC;
 147         else
 148                 zone = KBASE_REG_ZONE_CUSTOM_VA;
 149
 150         reg = kbase_alloc_free_region(kctx, 0, va_pages, zone);
 151         if (!reg) {
 152                 dev_err(dev, "Failed to allocate free region");
 153                 goto no_region;
 154         }
 155
 156         kbase_update_region_flags(kctx, reg, *flags);
 157
 158         if (kbase_reg_prepare_native(reg, kctx) != 0) {
 159                 dev_err(dev, "Failed to prepare region");
 160                 goto prepare_failed;
 161         }
 162
 163         if (*flags & BASE_MEM_GROW_ON_GPF)
 164                 reg->extent = extent;
 165         else
 166                 reg->extent = 0;
 167
 168         if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) {
 169                 dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
 170                                 (unsigned long long)commit_pages,
 171                                 (unsigned long long)va_pages);
 172                 goto no_mem;
 173         }
 174
 175         kbase_gpu_vm_lock(kctx);
 176
 177         /* mmap needed to setup VA? */
 178         if (*flags & BASE_MEM_SAME_VA) {
 179                 unsigned long prot = PROT_NONE;
 180                 unsigned long va_size = va_pages << PAGE_SHIFT;
 181                 unsigned long va_map = va_size;
 182                 unsigned long cookie, cookie_nr;
 183                 unsigned long cpu_addr;
 184
 185                 /* Bind to a cookie */
 186                 if (!kctx->cookies) {
 187                         dev_err(dev, "No cookies available for allocation!");
 188                         kbase_gpu_vm_unlock(kctx);
 189                         goto no_cookie;
 190                 }
 191                 /* return a cookie */
 192                 cookie_nr = __ffs(kctx->cookies);
 193                 kctx->cookies &= ~(1UL << cookie_nr);
 194                 BUG_ON(kctx->pending_regions[cookie_nr]);
 195                 kctx->pending_regions[cookie_nr] = reg;
 196
 197                 kbase_gpu_vm_unlock(kctx);
 198
 199                 /* relocate to correct base */
 200                 cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE);
 201                 cookie <<= PAGE_SHIFT;
 202
 203                 /* See if we must align memory due to GPU PC bits vs CPU VA */
 204                 if ((*flags & BASE_MEM_PROT_GPU_EX) &&
 205                     (cpu_va_bits > gpu_pc_bits)) {
 206                         *va_alignment = gpu_pc_bits;
 207                         reg->flags |= KBASE_REG_ALIGNED;
 208                 }
 209
 210                 /*
 211                  * Pre-10.1 UKU userland calls mmap for us so return the
 212                  * unaligned address and skip the map.
 213                  */
 214                 if (kctx->api_version < KBASE_API_VERSION(10, 1)) {
 215                         *gpu_va = (u64) cookie;
 216                         return reg;
 217                 }
 218
 219                 /*
 220                  * GPUCORE-2190:
 221                  *
 222                  * We still need to return alignment for old userspace.
 223                  */
 224                 if (*va_alignment)
 225                         va_map += 3 * (1UL << *va_alignment);
 226
 227                 if (*flags & BASE_MEM_PROT_CPU_RD)
 228                         prot |= PROT_READ;
 229                 if (*flags & BASE_MEM_PROT_CPU_WR)
 230                         prot |= PROT_WRITE;
 231
 232                 cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot,
 233                                 MAP_SHARED, cookie);
 234
 235                 if (IS_ERR_VALUE(cpu_addr)) {
 236                         kctx->pending_regions[cookie_nr] = NULL;
 237                         kctx->cookies |= (1UL << cookie_nr);
 238                         goto no_mmap;
 239                 }
 240
 241                 /*
 242                  * If we had to allocate extra VA space to force the
 243                  * alignment release it.
 244                  */
 245                 if (*va_alignment) {
 246                         unsigned long alignment = 1UL << *va_alignment;
 247                         unsigned long align_mask = alignment - 1;
 248                         unsigned long addr;
 249                         unsigned long addr_end;
 250                         unsigned long aligned_addr;
 251                         unsigned long aligned_addr_end;
 252
 253                         addr = cpu_addr;
 254                         addr_end = addr + va_map;
 255
 256                         aligned_addr = (addr + align_mask) &
 257                                         ~((u64) align_mask);
 258                         aligned_addr_end = aligned_addr + va_size;
 259
 260                         if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
 261                                 /*
 262                                  * Can't end at 4GB boundary on some GPUs as
 263                                  * it will halt the shader.
 264                                  */
 265                                 aligned_addr += 2 * alignment;
 266                                 aligned_addr_end += 2 * alignment;
 267                         } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
 268                                 /*
 269                                  * Can't start at 4GB boundary on some GPUs as
 270                                  * it will halt the shader.
 271                                  */
 272                                 aligned_addr += alignment;
 273                                 aligned_addr_end += alignment;
 274                         }
 275
 276                         /* anything to chop off at the start? */
 277                         if (addr != aligned_addr)
 278                                 vm_munmap(addr, aligned_addr - addr);
 279
 280                         /* anything at the end? */
 281                         if (addr_end != aligned_addr_end)
 282                                 vm_munmap(aligned_addr_end,
 283                                                 addr_end - aligned_addr_end);
 284
 285                         *gpu_va = (u64) aligned_addr;
 286                 } else
 287                         *gpu_va = (u64) cpu_addr;
 288         } else /* we control the VA */ {
 289                 if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) {
 290                         dev_warn(dev, "Failed to map memory on GPU");
 291                         kbase_gpu_vm_unlock(kctx);
 292                         goto no_mmap;
 293                 }
 294                 /* return real GPU VA */
 295                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
 296
 297                 kbase_gpu_vm_unlock(kctx);
 298         }
 299
 300         return reg;
 301
 302 no_mmap:
 303 no_cookie:
 304 no_mem:
 305         kbase_mem_phy_alloc_put(reg->cpu_alloc);
 306         kbase_mem_phy_alloc_put(reg->gpu_alloc);
 307 prepare_failed:
 308         kfree(reg);
 309 no_region:
 310 bad_ex_size:
 311 bad_flags:
 312 bad_size:
 313         return NULL;
 314 }
 315 KBASE_EXPORT_TEST_API(kbase_mem_alloc);
 316
 317 int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out)
 318 {
 319         struct kbase_va_region *reg;
 320         int ret = -EINVAL;
 321
 322         KBASE_DEBUG_ASSERT(kctx);
 323         KBASE_DEBUG_ASSERT(out);
 324
 325         kbase_gpu_vm_lock(kctx);
 326
 327         /* Validate the region */
 328         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
 329         if (!reg || (reg->flags & KBASE_REG_FREE))
 330                 goto out_unlock;
 331
 332         switch (query) {
 333         case KBASE_MEM_QUERY_COMMIT_SIZE:
 334                 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) {
 335                         *out = kbase_reg_current_backed_size(reg);
 336                 } else {
 337                         size_t i;
 338                         struct kbase_aliased *aliased;
 339                         *out = 0;
 340                         aliased = reg->cpu_alloc->imported.alias.aliased;
 341                         for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++)
 342                                 *out += aliased[i].length;
 343                 }
 344                 break;
 345         case KBASE_MEM_QUERY_VA_SIZE:
 346                 *out = reg->nr_pages;
 347                 break;
 348         case KBASE_MEM_QUERY_FLAGS:
 349         {
 350                 *out = 0;
 351                 if (KBASE_REG_CPU_WR & reg->flags)
 352                         *out |= BASE_MEM_PROT_CPU_WR;
 353                 if (KBASE_REG_CPU_RD & reg->flags)
 354                         *out |= BASE_MEM_PROT_CPU_RD;
 355                 if (KBASE_REG_CPU_CACHED & reg->flags)
 356                         *out |= BASE_MEM_CACHED_CPU;
 357                 if (KBASE_REG_GPU_WR & reg->flags)
 358                         *out |= BASE_MEM_PROT_GPU_WR;
 359                 if (KBASE_REG_GPU_RD & reg->flags)
 360                         *out |= BASE_MEM_PROT_GPU_RD;
 361                 if (!(KBASE_REG_GPU_NX & reg->flags))
 362                         *out |= BASE_MEM_PROT_GPU_EX;
 363                 if (KBASE_REG_SHARE_BOTH & reg->flags)
 364                         *out |= BASE_MEM_COHERENT_SYSTEM;
 365                 if (KBASE_REG_SHARE_IN & reg->flags)
 366                         *out |= BASE_MEM_COHERENT_LOCAL;
 367                 break;
 368         }
 369         default:
 370                 *out = 0;
 371                 goto out_unlock;
 372         }
 373
 374         ret = 0;
 375
 376 out_unlock:
 377         kbase_gpu_vm_unlock(kctx);
 378         return ret;
 379 }
 380
 381 /**
 382  * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the
 383  * Ephemeral memory eviction list.
 384  * @s:        Shrinker
 385  * @sc:       Shrinker control
 386  *
 387  * Return: Number of pages which can be freed.
 388  */
 389 static
 390 unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s,
 391                 struct shrink_control *sc)
 392 {
 393         struct kbase_context *kctx;
 394         struct kbase_mem_phy_alloc *alloc;
 395         unsigned long pages = 0;
 396
 397         kctx = container_of(s, struct kbase_context, reclaim);
 398
 399         mutex_lock(&kctx->evict_lock);
 400
 401         list_for_each_entry(alloc, &kctx->evict_list, evict_node)
 402                 pages += alloc->nents;
 403
 404         mutex_unlock(&kctx->evict_lock);
 405         return pages;
 406 }
 407
 408 /**
 409  * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction
 410  * list for pages and try to reclaim them.
 411  * @s:        Shrinker
 412  * @sc:       Shrinker control
 413  *
 414  * Return: Number of pages freed (can be less then requested) or -1 if the
 415  * shrinker failed to free pages in its pool.
 416  *
 417  * Note:
 418  * This function accesses region structures without taking the region lock,
 419  * this is required as the OOM killer can call the shrinker after the region
 420  * lock has already been held.
 421  * This is safe as we can guarantee that a region on the eviction list will
 422  * not be freed (kbase_mem_free_region removes the allocation from the list
 423  * before destroying it), or modified by other parts of the driver.
 424  * The eviction list itself is guarded by the eviction lock and the MMU updates
 425  * are protected by their own lock.
 426  */
 427 static
 428 unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
 429                 struct shrink_control *sc)
 430 {
 431         struct kbase_context *kctx;
 432         struct kbase_mem_phy_alloc *alloc;
 433         struct kbase_mem_phy_alloc *tmp;
 434         unsigned long freed = 0;
 435
 436         kctx = container_of(s, struct kbase_context, reclaim);
 437         mutex_lock(&kctx->evict_lock);
 438
 439         list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
 440                 int err;
 441
 442                 err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
 443                                 0, alloc->nents);
 444                 if (err != 0) {
 445                         /*
 446                          * Failed to remove GPU mapping, tell the shrinker
 447                          * to stop trying to shrink our slab even though we
 448                          * have pages in it.
 449                          */
 450                         freed = -1;
 451                         goto out_unlock;
 452                 }
 453
 454                 /*
 455                  * Update alloc->evicted before freeing the backing so the
 456                  * helper can determine that it needs to bypass the accounting
 457                  * and memory pool.
 458                  */
 459                 alloc->evicted = alloc->nents;
 460
 461                 kbase_free_phy_pages_helper(alloc, alloc->evicted);
 462                 freed += alloc->evicted;
 463                 list_del_init(&alloc->evict_node);
 464
 465                 /*
 466                  * Inform the JIT allocator this region has lost backing
 467                  * as it might need to free the allocation.
 468                  */
 469                 kbase_jit_backing_lost(alloc->reg);
 470
 471                 /* Enough pages have been freed so stop now */
 472                 if (freed > sc->nr_to_scan)
 473                         break;
 474         }
 475 out_unlock:
 476         mutex_unlock(&kctx->evict_lock);
 477
 478         return freed;
 479 }
 480
 481 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 482 static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s,
 483                 struct shrink_control *sc)
 484 {
 485         if (sc->nr_to_scan == 0)
 486                 return kbase_mem_evictable_reclaim_count_objects(s, sc);
 487
 488         return kbase_mem_evictable_reclaim_scan_objects(s, sc);
 489 }
 490 #endif
 491
 492 int kbase_mem_evictable_init(struct kbase_context *kctx)
 493 {
 494         INIT_LIST_HEAD(&kctx->evict_list);
 495         mutex_init(&kctx->evict_lock);
 496
 497         /* Register shrinker */
 498 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 499         kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink;
 500 #else
 501         kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects;
 502         kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects;
 503 #endif
 504         kctx->reclaim.seeks = DEFAULT_SEEKS;
 505         /* Kernel versions prior to 3.1 :
 506          * struct shrinker does not define batch */
 507 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
 508         kctx->reclaim.batch = 0;
 509 #endif
 510         register_shrinker(&kctx->reclaim);
 511         return 0;
 512 }
 513
 514 void kbase_mem_evictable_deinit(struct kbase_context *kctx)
 515 {
 516         unregister_shrinker(&kctx->reclaim);
 517 }
 518
 519 struct kbase_mem_zone_cache_entry {
 520         /* List head used to link the cache entry to the memory allocation. */
 521         struct list_head zone_node;
 522         /* The zone the cacheline is for. */
 523         struct zone *zone;
 524         /* The number of pages in the allocation which belong to this zone. */
 525         u64 count;
 526 };
 527
 528 static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc,
 529                 size_t start_offset)
 530 {
 531         struct kbase_mem_zone_cache_entry *cache = NULL;
 532         size_t i;
 533         int ret = 0;
 534
 535         for (i = start_offset; i < alloc->nents; i++) {
 536                 struct page *p = phys_to_page(alloc->pages[i]);
 537                 struct zone *zone = page_zone(p);
 538                 bool create = true;
 539
 540                 if (cache && (cache->zone == zone)) {
 541                         /*
 542                          * Fast path check as most of the time adjacent
 543                          * pages come from the same zone.
 544                          */
 545                         create = false;
 546                 } else {
 547                         /*
 548                          * Slow path check, walk all the cache entries to see
 549                          * if we already know about this zone.
 550                          */
 551                         list_for_each_entry(cache, &alloc->zone_cache, zone_node) {
 552                                 if (cache->zone == zone) {
 553                                         create = false;
 554                                         break;
 555                                 }
 556                         }
 557                 }
 558
 559                 /* This zone wasn't found in the cache, create an entry for it */
 560                 if (create) {
 561                         cache = kmalloc(sizeof(*cache), GFP_KERNEL);
 562                         if (!cache) {
 563                                 ret = -ENOMEM;
 564                                 goto bail;
 565                         }
 566                         cache->zone = zone;
 567                         cache->count = 0;
 568                         list_add(&cache->zone_node, &alloc->zone_cache);
 569                 }
 570
 571                 cache->count++;
 572         }
 573         return 0;
 574
 575 bail:
 576         return ret;
 577 }
 578
 579 int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
 580                 size_t start_offset)
 581 {
 582         /*
 583          * Bail if the zone cache is empty, only update the cache if it
 584          * existed in the first place.
 585          */
 586         if (list_empty(&alloc->zone_cache))
 587                 return 0;
 588
 589         return kbase_zone_cache_builder(alloc, start_offset);
 590 }
 591
 592 int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc)
 593 {
 594         /* Bail if the zone cache already exists */
 595         if (!list_empty(&alloc->zone_cache))
 596                 return 0;
 597
 598         return kbase_zone_cache_builder(alloc, 0);
 599 }
 600
 601 void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc)
 602 {
 603         struct kbase_mem_zone_cache_entry *walker;
 604
 605         while(!list_empty(&alloc->zone_cache)){
 606                 walker = list_first_entry(&alloc->zone_cache,
 607                                 struct kbase_mem_zone_cache_entry, zone_node);
 608                 list_del(&walker->zone_node);
 609                 kfree(walker);
 610         }
 611 }
 612
 613 /**
 614  * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
 615  * @alloc: The physical allocation
 616  */
 617 static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc)
 618 {
 619         struct kbase_context *kctx = alloc->imported.kctx;
 620         struct kbase_mem_zone_cache_entry *zone_cache;
 621         int __maybe_unused new_page_count;
 622         int err;
 623
 624         /* Attempt to build a zone cache of tracking */
 625         err = kbase_zone_cache_build(alloc);
 626         if (err == 0) {
 627                 /* Bulk update all the zones */
 628                 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
 629                         zone_page_state_add(zone_cache->count,
 630                                         zone_cache->zone, NR_SLAB_RECLAIMABLE);
 631                 }
 632         } else {
 633                 /* Fall-back to page by page updates */
 634                 int i;
 635
 636                 for (i = 0; i < alloc->nents; i++) {
 637                         struct page *p = phys_to_page(alloc->pages[i]);
 638                         struct zone *zone = page_zone(p);
 639
 640                         zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE);
 641                 }
 642         }
 643
 644         kbase_process_page_usage_dec(kctx, alloc->nents);
 645         new_page_count = kbase_atomic_sub_pages(alloc->nents,
 646                                                 &kctx->used_pages);
 647         kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
 648
 649         kbase_tlstream_aux_pagesalloc(
 650                         (u32)kctx->id,
 651                         (u64)new_page_count);
 652 }
 653
 654 /**
 655  * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable.
 656  * @alloc: The physical allocation
 657  */
 658 static
 659 void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc)
 660 {
 661         struct kbase_context *kctx = alloc->imported.kctx;
 662         struct kbase_mem_zone_cache_entry *zone_cache;
 663         int __maybe_unused new_page_count;
 664         int err;
 665
 666         new_page_count = kbase_atomic_add_pages(alloc->nents,
 667                                                 &kctx->used_pages);
 668         kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
 669
 670         /* Increase mm counters so that the allocation is accounted for
 671          * against the process and thus is visible to the OOM killer,
 672          * then remove it from the reclaimable accounting. */
 673         kbase_process_page_usage_inc(kctx, alloc->nents);
 674
 675         /* Attempt to build a zone cache of tracking */
 676         err = kbase_zone_cache_build(alloc);
 677         if (err == 0) {
 678                 /* Bulk update all the zones */
 679                 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
 680                         zone_page_state_add(-zone_cache->count,
 681                                         zone_cache->zone, NR_SLAB_RECLAIMABLE);
 682                 }
 683         } else {
 684                 /* Fall-back to page by page updates */
 685                 int i;
 686
 687                 for (i = 0; i < alloc->nents; i++) {
 688                         struct page *p = phys_to_page(alloc->pages[i]);
 689                         struct zone *zone = page_zone(p);
 690
 691                         zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE);
 692                 }
 693         }
 694
 695         kbase_tlstream_aux_pagesalloc(
 696                         (u32)kctx->id,
 697                         (u64)new_page_count);
 698 }
 699
 700 int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc)
 701 {
 702         struct kbase_context *kctx = gpu_alloc->imported.kctx;
 703         int err;
 704
 705         lockdep_assert_held(&kctx->reg_lock);
 706
 707         /* This alloction can't already be on a list. */
 708         WARN_ON(!list_empty(&gpu_alloc->evict_node));
 709
 710         /*
 711          * Try to shrink the CPU mappings as required, if we fail then
 712          * fail the process of making this allocation evictable.
 713          */
 714         err = kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg,
 715                         0, gpu_alloc->nents);
 716         if (err)
 717                 return -EINVAL;
 718
 719         /*
 720          * Add the allocation to the eviction list, after this point the shrink
 721          * can reclaim it.
 722          */
 723         mutex_lock(&kctx->evict_lock);
 724         list_add(&gpu_alloc->evict_node, &kctx->evict_list);
 725         mutex_unlock(&kctx->evict_lock);
 726         kbase_mem_evictable_mark_reclaim(gpu_alloc);
 727
 728         gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED;
 729         return 0;
 730 }
 731
 732 bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc)
 733 {
 734         struct kbase_context *kctx = gpu_alloc->imported.kctx;
 735         int err = 0;
 736
 737         lockdep_assert_held(&kctx->reg_lock);
 738
 739         /*
 740          * First remove the allocation from the eviction list as it's no
 741          * longer eligible for eviction.
 742          */
 743         mutex_lock(&kctx->evict_lock);
 744         list_del_init(&gpu_alloc->evict_node);
 745         mutex_unlock(&kctx->evict_lock);
 746
 747         if (gpu_alloc->evicted == 0) {
 748                 /*
 749                  * The backing is still present, update the VM stats as it's
 750                  * in use again.
 751                  */
 752                 kbase_mem_evictable_unmark_reclaim(gpu_alloc);
 753         } else {
 754                 /* If the region is still alive ... */
 755                 if (gpu_alloc->reg) {
 756                         /* ... allocate replacement backing ... */
 757                         err = kbase_alloc_phy_pages_helper(gpu_alloc,
 758                                         gpu_alloc->evicted);
 759
 760                         /*
 761                          * ... and grow the mapping back to its
 762                          * pre-eviction size.
 763                          */
 764                         if (!err)
 765                                 err = kbase_mem_grow_gpu_mapping(kctx,
 766                                                 gpu_alloc->reg,
 767                                                 gpu_alloc->evicted, 0);
 768
 769                         gpu_alloc->evicted = 0;
 770                 }
 771         }
 772
 773         /* If the region is still alive remove the DONT_NEED attribute. */
 774         if (gpu_alloc->reg)
 775                 gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED;
 776
 777         return (err == 0);
 778 }
 779
 780 int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask)
 781 {
 782         struct kbase_va_region *reg;
 783         int ret = -EINVAL;
 784         unsigned int real_flags = 0;
 785         unsigned int prev_flags = 0;
 786         bool prev_needed, new_needed;
 787
 788         KBASE_DEBUG_ASSERT(kctx);
 789
 790         if (!gpu_addr)
 791                 return -EINVAL;
 792
 793         /* nuke other bits */
 794         flags &= mask;
 795
 796         /* check for only supported flags */
 797         if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE))
 798                 goto out;
 799
 800         /* mask covers bits we don't support? */
 801         if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE))
 802                 goto out;
 803
 804         /* convert flags */
 805         if (BASE_MEM_COHERENT_SYSTEM & flags)
 806                 real_flags |= KBASE_REG_SHARE_BOTH;
 807         else if (BASE_MEM_COHERENT_LOCAL & flags)
 808                 real_flags |= KBASE_REG_SHARE_IN;
 809
 810         /* now we can lock down the context, and find the region */
 811         down_write(&current->mm->mmap_sem);
 812         kbase_gpu_vm_lock(kctx);
 813
 814         /* Validate the region */
 815         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
 816         if (!reg || (reg->flags & KBASE_REG_FREE))
 817                 goto out_unlock;
 818
 819         /* Is the region being transitioning between not needed and needed? */
 820         prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED;
 821         new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED;
 822         if (prev_needed != new_needed) {
 823                 /* Aliased allocations can't be made ephemeral */
 824                 if (atomic_read(&reg->cpu_alloc->gpu_mappings) > 1)
 825                         goto out_unlock;
 826
 827                 if (new_needed) {
 828                         /* Only native allocations can be marked not needed */
 829                         if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
 830                                 ret = -EINVAL;
 831                                 goto out_unlock;
 832                         }
 833                         ret = kbase_mem_evictable_make(reg->gpu_alloc);
 834                         if (ret)
 835                                 goto out_unlock;
 836                 } else {
 837                         kbase_mem_evictable_unmake(reg->gpu_alloc);
 838                 }
 839         }
 840
 841         /* limit to imported memory */
 842         if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
 843              (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
 844                 goto out_unlock;
 845
 846         /* no change? */
 847         if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) {
 848                 ret = 0;
 849                 goto out_unlock;
 850         }
 851
 852         /* save for roll back */
 853         prev_flags = reg->flags;
 854         reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH);
 855         reg->flags |= real_flags;
 856
 857         /* Currently supporting only imported memory */
 858         switch (reg->gpu_alloc->type) {
 859 #ifdef CONFIG_UMP
 860         case KBASE_MEM_TYPE_IMPORTED_UMP:
 861                 ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags);
 862                 break;
 863 #endif
 864 #ifdef CONFIG_DMA_SHARED_BUFFER
 865         case KBASE_MEM_TYPE_IMPORTED_UMM:
 866                 /* Future use will use the new flags, existing mapping will NOT be updated
 867                  * as memory should not be in use by the GPU when updating the flags.
 868                  */
 869                 ret = 0;
 870                 WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count);
 871                 break;
 872 #endif
 873         default:
 874                 break;
 875         }
 876
 877         /* roll back on error, i.e. not UMP */
 878         if (ret)
 879                 reg->flags = prev_flags;
 880
 881 out_unlock:
 882         kbase_gpu_vm_unlock(kctx);
 883         up_write(&current->mm->mmap_sem);
 884 out:
 885         return ret;
 886 }
 887
 888 #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS)
 889
 890 #ifdef CONFIG_UMP
 891 static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags)
 892 {
 893         struct kbase_va_region *reg;
 894         ump_dd_handle umph;
 895         u64 block_count;
 896         const ump_dd_physical_block_64 *block_array;
 897         u64 i, j;
 898         int page = 0;
 899         ump_alloc_flags ump_flags;
 900         ump_alloc_flags cpu_flags;
 901         ump_alloc_flags gpu_flags;
 902
 903         if (*flags & BASE_MEM_SECURE)
 904                 goto bad_flags;
 905
 906         umph = ump_dd_from_secure_id(id);
 907         if (UMP_DD_INVALID_MEMORY_HANDLE == umph)
 908                 goto bad_id;
 909
 910         ump_flags = ump_dd_allocation_flags_get(umph);
 911         cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK;
 912         gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) &
 913                         UMP_DEVICE_MASK;
 914
 915         *va_pages = ump_dd_size_get_64(umph);
 916         *va_pages >>= PAGE_SHIFT;
 917
 918         if (!*va_pages)
 919                 goto bad_size;
 920
 921         if (*va_pages > (U64_MAX / PAGE_SIZE))
 922                 /* 64-bit address range is the max */
 923                 goto bad_size;
 924
 925         if (*flags & BASE_MEM_SAME_VA)
 926                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
 927         else
 928                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
 929
 930         if (!reg)
 931                 goto no_region;
 932
 933         /* we've got pages to map now, and support SAME_VA */
 934         *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
 935
 936         reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
 937         if (IS_ERR_OR_NULL(reg->gpu_alloc))
 938                 goto no_alloc_obj;
 939
 940         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
 941
 942         reg->gpu_alloc->imported.ump_handle = umph;
 943
 944         reg->flags &= ~KBASE_REG_FREE;
 945         reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */
 946         reg->flags &= ~KBASE_REG_GROWABLE;      /* UMP cannot be grown */
 947
 948         /* Override import flags based on UMP flags */
 949         *flags &= ~(BASE_MEM_CACHED_CPU);
 950         *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR);
 951         *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR);
 952
 953         if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
 954             (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) {
 955                 reg->flags |= KBASE_REG_CPU_CACHED;
 956                 *flags |= BASE_MEM_CACHED_CPU;
 957         }
 958
 959         if (cpu_flags & UMP_PROT_CPU_WR) {
 960                 reg->flags |= KBASE_REG_CPU_WR;
 961                 *flags |= BASE_MEM_PROT_CPU_WR;
 962         }
 963
 964         if (cpu_flags & UMP_PROT_CPU_RD) {
 965                 reg->flags |= KBASE_REG_CPU_RD;
 966                 *flags |= BASE_MEM_PROT_CPU_RD;
 967         }
 968
 969         if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
 970             (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR))
 971                 reg->flags |= KBASE_REG_GPU_CACHED;
 972
 973         if (gpu_flags & UMP_PROT_DEVICE_WR) {
 974                 reg->flags |= KBASE_REG_GPU_WR;
 975                 *flags |= BASE_MEM_PROT_GPU_WR;
 976         }
 977
 978         if (gpu_flags & UMP_PROT_DEVICE_RD) {
 979                 reg->flags |= KBASE_REG_GPU_RD;
 980                 *flags |= BASE_MEM_PROT_GPU_RD;
 981         }
 982
 983         /* ump phys block query */
 984         ump_dd_phys_blocks_get_64(umph, &block_count, &block_array);
 985
 986         for (i = 0; i < block_count; i++) {
 987                 for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) {
 988                         reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
 989                         page++;
 990                 }
 991         }
 992         reg->gpu_alloc->nents = *va_pages;
 993         reg->extent = 0;
 994
 995         return reg;
 996
 997 no_alloc_obj:
 998         kfree(reg);
 999 no_region:
1000 bad_size:
1001         ump_dd_release(umph);
1002 bad_id:
1003 bad_flags:
1004         return NULL;
1005 }
1006 #endif                          /* CONFIG_UMP */
1007
1008 #ifdef CONFIG_DMA_SHARED_BUFFER
1009 static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags)
1010 {
1011         struct kbase_va_region *reg;
1012         struct dma_buf *dma_buf;
1013         struct dma_buf_attachment *dma_attachment;
1014         bool shared_zone = false;
1015
1016         dma_buf = dma_buf_get(fd);
1017         if (IS_ERR_OR_NULL(dma_buf))
1018                 goto no_buf;
1019
1020         dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev);
1021         if (!dma_attachment)
1022                 goto no_attachment;
1023
1024         *va_pages = PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT;
1025         if (!*va_pages)
1026                 goto bad_size;
1027
1028         if (*va_pages > (U64_MAX / PAGE_SIZE))
1029                 /* 64-bit address range is the max */
1030                 goto bad_size;
1031
1032         /* ignore SAME_VA */
1033         *flags &= ~BASE_MEM_SAME_VA;
1034
1035         if (*flags & BASE_MEM_IMPORT_SHARED)
1036                 shared_zone = true;
1037
1038 #ifdef CONFIG_64BIT
1039         if (!kctx->is_compat) {
1040                 /*
1041                  * 64-bit tasks require us to reserve VA on the CPU that we use
1042                  * on the GPU.
1043                  */
1044                 shared_zone = true;
1045         }
1046 #endif
1047
1048         if (shared_zone) {
1049                 *flags |= BASE_MEM_NEED_MMAP;
1050                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
1051         } else {
1052                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
1053         }
1054
1055         if (!reg)
1056                 goto no_region;
1057
1058         reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
1059         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1060                 goto no_alloc_obj;
1061
1062         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1063
1064         /* No pages to map yet */
1065         reg->gpu_alloc->nents = 0;
1066
1067         reg->flags &= ~KBASE_REG_FREE;
1068         reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */
1069         reg->flags &= ~KBASE_REG_GROWABLE;      /* UMM cannot be grown */
1070         reg->flags |= KBASE_REG_GPU_CACHED;
1071
1072         if (*flags & BASE_MEM_PROT_CPU_WR)
1073                 reg->flags |= KBASE_REG_CPU_WR;
1074
1075         if (*flags & BASE_MEM_PROT_CPU_RD)
1076                 reg->flags |= KBASE_REG_CPU_RD;
1077
1078         if (*flags & BASE_MEM_PROT_GPU_WR)
1079                 reg->flags |= KBASE_REG_GPU_WR;
1080
1081         if (*flags & BASE_MEM_PROT_GPU_RD)
1082                 reg->flags |= KBASE_REG_GPU_RD;
1083
1084         if (*flags & BASE_MEM_SECURE)
1085                 reg->flags |= KBASE_REG_SECURE;
1086
1087         /* no read or write permission given on import, only on run do we give the right permissions */
1088
1089         reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM;
1090         reg->gpu_alloc->imported.umm.sgt = NULL;
1091         reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
1092         reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment;
1093         reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0;
1094         reg->extent = 0;
1095
1096         return reg;
1097
1098 no_alloc_obj:
1099         kfree(reg);
1100 no_region:
1101 bad_size:
1102         dma_buf_detach(dma_buf, dma_attachment);
1103 no_attachment:
1104         dma_buf_put(dma_buf);
1105 no_buf:
1106         return NULL;
1107 }
1108 #endif  /* CONFIG_DMA_SHARED_BUFFER */
1109
1110
1111 static struct kbase_va_region *kbase_mem_from_user_buffer(
1112                 struct kbase_context *kctx, unsigned long address,
1113                 unsigned long size, u64 *va_pages, u64 *flags)
1114 {
1115         struct kbase_va_region *reg;
1116         long faulted_pages;
1117         int zone = KBASE_REG_ZONE_CUSTOM_VA;
1118         bool shared_zone = false;
1119
1120         *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) -
1121                 PFN_DOWN(address);
1122         if (!*va_pages)
1123                 goto bad_size;
1124
1125         if (*va_pages > (UINT64_MAX / PAGE_SIZE))
1126                 /* 64-bit address range is the max */
1127                 goto bad_size;
1128
1129         /* SAME_VA generally not supported with imported memory (no known use cases) */
1130         *flags &= ~BASE_MEM_SAME_VA;
1131
1132         if (*flags & BASE_MEM_IMPORT_SHARED)
1133                 shared_zone = true;
1134
1135 #ifdef CONFIG_64BIT
1136         if (!kctx->is_compat) {
1137                 /*
1138                  * 64-bit tasks require us to reserve VA on the CPU that we use
1139                  * on the GPU.
1140                  */
1141                 shared_zone = true;
1142         }
1143 #endif
1144
1145         if (shared_zone) {
1146                 *flags |= BASE_MEM_NEED_MMAP;
1147                 zone = KBASE_REG_ZONE_SAME_VA;
1148         }
1149
1150         reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone);
1151
1152         if (!reg)
1153                 goto no_region;
1154
1155         reg->gpu_alloc = kbase_alloc_create(*va_pages,
1156                         KBASE_MEM_TYPE_IMPORTED_USER_BUF);
1157         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1158                 goto no_alloc_obj;
1159
1160         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1161
1162         reg->flags &= ~KBASE_REG_FREE;
1163         reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */
1164         reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */
1165
1166         if (*flags & BASE_MEM_PROT_CPU_WR)
1167                 reg->flags |= KBASE_REG_CPU_WR;
1168
1169         if (*flags & BASE_MEM_PROT_CPU_RD)
1170                 reg->flags |= KBASE_REG_CPU_RD;
1171
1172         if (*flags & BASE_MEM_PROT_GPU_WR)
1173                 reg->flags |= KBASE_REG_GPU_WR;
1174
1175         if (*flags & BASE_MEM_PROT_GPU_RD)
1176                 reg->flags |= KBASE_REG_GPU_RD;
1177
1178         down_read(&current->mm->mmap_sem);
1179
1180         /* A sanity check that get_user_pages will work on the memory */
1181         /* (so the initial import fails on weird memory regions rather than */
1182         /* the job failing when we try to handle the external resources). */
1183         /* It doesn't take a reference to the pages (because the page list is NULL). */
1184         /* We can't really store the page list because that would involve */
1185         /* keeping the pages pinned - instead we pin/unpin around the job */
1186         /* (as part of the external resources handling code) */
1187 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1188         faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
1189                         reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1190 #else
1191         faulted_pages = get_user_pages(address, *va_pages,
1192                         reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1193 #endif
1194         up_read(&current->mm->mmap_sem);
1195
1196         if (faulted_pages != *va_pages)
1197                 goto fault_mismatch;
1198
1199         reg->gpu_alloc->imported.user_buf.size = size;
1200         reg->gpu_alloc->imported.user_buf.address = address;
1201         reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages;
1202         reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages,
1203                         sizeof(struct page *), GFP_KERNEL);
1204         reg->gpu_alloc->imported.user_buf.mm = current->mm;
1205         atomic_inc(&current->mm->mm_count);
1206
1207         if (!reg->gpu_alloc->imported.user_buf.pages)
1208                 goto no_page_array;
1209
1210         reg->gpu_alloc->nents = 0;
1211         reg->extent = 0;
1212
1213         return reg;
1214
1215 no_page_array:
1216 fault_mismatch:
1217         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1218 no_alloc_obj:
1219         kfree(reg);
1220 no_region:
1221 bad_size:
1222         return NULL;
1223
1224 }
1225
1226
1227 u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride,
1228                     u64 nents, struct base_mem_aliasing_info *ai,
1229                     u64 *num_pages)
1230 {
1231         struct kbase_va_region *reg;
1232         u64 gpu_va;
1233         size_t i;
1234         bool coherent;
1235
1236         KBASE_DEBUG_ASSERT(kctx);
1237         KBASE_DEBUG_ASSERT(flags);
1238         KBASE_DEBUG_ASSERT(ai);
1239         KBASE_DEBUG_ASSERT(num_pages);
1240
1241         /* mask to only allowed flags */
1242         *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
1243                    BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL |
1244                    BASE_MEM_COHERENT_SYSTEM_REQUIRED);
1245
1246         if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
1247                 dev_warn(kctx->kbdev->dev,
1248                                 "kbase_mem_alias called with bad flags (%llx)",
1249                                 (unsigned long long)*flags);
1250                 goto bad_flags;
1251         }
1252         coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 ||
1253                         (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0;
1254
1255         if (!stride)
1256                 goto bad_stride;
1257
1258         if (!nents)
1259                 goto bad_nents;
1260
1261         if ((nents * stride) > (U64_MAX / PAGE_SIZE))
1262                 /* 64-bit address range is the max */
1263                 goto bad_size;
1264
1265         /* calculate the number of pages this alias will cover */
1266         *num_pages = nents * stride;
1267
1268 #ifdef CONFIG_64BIT
1269         if (!kctx->is_compat) {
1270                 /* 64-bit tasks must MMAP anyway, but not expose this address to
1271                  * clients */
1272                 *flags |= BASE_MEM_NEED_MMAP;
1273                 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1274                                               KBASE_REG_ZONE_SAME_VA);
1275         } else {
1276 #else
1277         if (1) {
1278 #endif
1279                 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1280                                               KBASE_REG_ZONE_CUSTOM_VA);
1281         }
1282
1283         if (!reg)
1284                 goto no_reg;
1285
1286         /* zero-sized page array, as we don't need one/can support one */
1287         reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
1288         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1289                 goto no_alloc_obj;
1290
1291         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1292
1293         kbase_update_region_flags(kctx, reg, *flags);
1294
1295         reg->gpu_alloc->imported.alias.nents = nents;
1296         reg->gpu_alloc->imported.alias.stride = stride;
1297         reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents);
1298         if (!reg->gpu_alloc->imported.alias.aliased)
1299                 goto no_aliased_array;
1300
1301         kbase_gpu_vm_lock(kctx);
1302
1303         /* validate and add src handles */
1304         for (i = 0; i < nents; i++) {
1305                 if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) {
1306                         if (ai[i].handle.basep.handle !=
1307                             BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE)
1308                                 goto bad_handle; /* unsupported magic handle */
1309                         if (!ai[i].length)
1310                                 goto bad_handle; /* must be > 0 */
1311                         if (ai[i].length > stride)
1312                                 goto bad_handle; /* can't be larger than the
1313                                                     stride */
1314                         reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1315                 } else {
1316                         struct kbase_va_region *aliasing_reg;
1317                         struct kbase_mem_phy_alloc *alloc;
1318
1319                         aliasing_reg = kbase_region_tracker_find_region_base_address(
1320                                 kctx,
1321                                 (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT);
1322
1323                         /* validate found region */
1324                         if (!aliasing_reg)
1325                                 goto bad_handle; /* Not found */
1326                         if (aliasing_reg->flags & KBASE_REG_FREE)
1327                                 goto bad_handle; /* Free region */
1328                         if (aliasing_reg->flags & KBASE_REG_DONT_NEED)
1329                                 goto bad_handle; /* Ephemeral region */
1330                         if (!aliasing_reg->gpu_alloc)
1331                                 goto bad_handle; /* No alloc */
1332                         if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
1333                                 goto bad_handle; /* Not a native alloc */
1334                         if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0))
1335                                 goto bad_handle;
1336                                 /* Non-coherent memory cannot alias
1337                                    coherent memory, and vice versa.*/
1338
1339                         /* check size against stride */
1340                         if (!ai[i].length)
1341                                 goto bad_handle; /* must be > 0 */
1342                         if (ai[i].length > stride)
1343                                 goto bad_handle; /* can't be larger than the
1344                                                     stride */
1345
1346                         alloc = aliasing_reg->gpu_alloc;
1347
1348                         /* check against the alloc's size */
1349                         if (ai[i].offset > alloc->nents)
1350                                 goto bad_handle; /* beyond end */
1351                         if (ai[i].offset + ai[i].length > alloc->nents)
1352                                 goto bad_handle; /* beyond end */
1353
1354                         reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
1355                         reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1356                         reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset;
1357                 }
1358         }
1359
1360 #ifdef CONFIG_64BIT
1361         if (!kctx->is_compat) {
1362                 /* Bind to a cookie */
1363                 if (!kctx->cookies) {
1364                         dev_err(kctx->kbdev->dev, "No cookies available for allocation!");
1365                         goto no_cookie;
1366                 }
1367                 /* return a cookie */
1368                 gpu_va = __ffs(kctx->cookies);
1369                 kctx->cookies &= ~(1UL << gpu_va);
1370                 BUG_ON(kctx->pending_regions[gpu_va]);
1371                 kctx->pending_regions[gpu_va] = reg;
1372
1373                 /* relocate to correct base */
1374                 gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1375                 gpu_va <<= PAGE_SHIFT;
1376         } else /* we control the VA */ {
1377 #else
1378         if (1) {
1379 #endif
1380                 if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
1381                         dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
1382                         goto no_mmap;
1383                 }
1384                 /* return real GPU VA */
1385                 gpu_va = reg->start_pfn << PAGE_SHIFT;
1386         }
1387
1388         reg->flags &= ~KBASE_REG_FREE;
1389         reg->flags &= ~KBASE_REG_GROWABLE;
1390
1391         kbase_gpu_vm_unlock(kctx);
1392
1393         return gpu_va;
1394
1395 #ifdef CONFIG_64BIT
1396 no_cookie:
1397 #endif
1398 no_mmap:
1399 bad_handle:
1400         kbase_gpu_vm_unlock(kctx);
1401 no_aliased_array:
1402         kbase_mem_phy_alloc_put(reg->cpu_alloc);
1403         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1404 no_alloc_obj:
1405         kfree(reg);
1406 no_reg:
1407 bad_size:
1408 bad_nents:
1409 bad_stride:
1410 bad_flags:
1411         return 0;
1412 }
1413
1414 int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
1415                 void __user *phandle, u64 *gpu_va, u64 *va_pages,
1416                 u64 *flags)
1417 {
1418         struct kbase_va_region *reg;
1419
1420         KBASE_DEBUG_ASSERT(kctx);
1421         KBASE_DEBUG_ASSERT(gpu_va);
1422         KBASE_DEBUG_ASSERT(va_pages);
1423         KBASE_DEBUG_ASSERT(flags);
1424
1425 #ifdef CONFIG_64BIT
1426         if (!kctx->is_compat)
1427                 *flags |= BASE_MEM_SAME_VA;
1428 #endif
1429
1430         if (!kbase_check_import_flags(*flags)) {
1431                 dev_warn(kctx->kbdev->dev,
1432                                 "kbase_mem_import called with bad flags (%llx)",
1433                                 (unsigned long long)*flags);
1434                 goto bad_flags;
1435         }
1436
1437         switch (type) {
1438 #ifdef CONFIG_UMP
1439         case BASE_MEM_IMPORT_TYPE_UMP: {
1440                 ump_secure_id id;
1441
1442                 if (get_user(id, (ump_secure_id __user *)phandle))
1443                         reg = NULL;
1444                 else
1445                         reg = kbase_mem_from_ump(kctx, id, va_pages, flags);
1446         }
1447         break;
1448 #endif /* CONFIG_UMP */
1449 #ifdef CONFIG_DMA_SHARED_BUFFER
1450         case BASE_MEM_IMPORT_TYPE_UMM: {
1451                 int fd;
1452
1453                 if (get_user(fd, (int __user *)phandle))
1454                         reg = NULL;
1455                 else
1456                         reg = kbase_mem_from_umm(kctx, fd, va_pages, flags);
1457         }
1458         break;
1459 #endif /* CONFIG_DMA_SHARED_BUFFER */
1460         case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
1461                 struct base_mem_import_user_buffer user_buffer;
1462                 void __user *uptr;
1463
1464                 if (copy_from_user(&user_buffer, phandle,
1465                                 sizeof(user_buffer))) {
1466                         reg = NULL;
1467                 } else {
1468 #ifdef CONFIG_COMPAT
1469                         if (kctx->is_compat)
1470                                 uptr = compat_ptr(user_buffer.ptr.compat_value);
1471                         else
1472 #endif
1473                                 uptr = user_buffer.ptr.value;
1474
1475                         reg = kbase_mem_from_user_buffer(kctx,
1476                                         (unsigned long)uptr, user_buffer.length,
1477                                         va_pages, flags);
1478                 }
1479                 break;
1480         }
1481         default: {
1482                 reg = NULL;
1483                 break;
1484         }
1485         }
1486
1487         if (!reg)
1488                 goto no_reg;
1489
1490         kbase_gpu_vm_lock(kctx);
1491
1492         /* mmap needed to setup VA? */
1493         if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) {
1494                 /* Bind to a cookie */
1495                 if (!kctx->cookies)
1496                         goto no_cookie;
1497                 /* return a cookie */
1498                 *gpu_va = __ffs(kctx->cookies);
1499                 kctx->cookies &= ~(1UL << *gpu_va);
1500                 BUG_ON(kctx->pending_regions[*gpu_va]);
1501                 kctx->pending_regions[*gpu_va] = reg;
1502
1503                 /* relocate to correct base */
1504                 *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1505                 *gpu_va <<= PAGE_SHIFT;
1506
1507         } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES)  {
1508                 /* we control the VA, mmap now to the GPU */
1509                 if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
1510                         goto no_gpu_va;
1511                 /* return real GPU VA */
1512                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1513         } else {
1514                 /* we control the VA, but nothing to mmap yet */
1515                 if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0)
1516                         goto no_gpu_va;
1517                 /* return real GPU VA */
1518                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1519         }
1520
1521         /* clear out private flags */
1522         *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1);
1523
1524         kbase_gpu_vm_unlock(kctx);
1525
1526         return 0;
1527
1528 no_gpu_va:
1529 no_cookie:
1530         kbase_gpu_vm_unlock(kctx);
1531         kbase_mem_phy_alloc_put(reg->cpu_alloc);
1532         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1533         kfree(reg);
1534 no_reg:
1535 bad_flags:
1536         *gpu_va = 0;
1537         *va_pages = 0;
1538         *flags = 0;
1539         return -ENOMEM;
1540 }
1541
1542
1543 static int zap_range_nolock(struct mm_struct *mm,
1544                 const struct vm_operations_struct *vm_ops,
1545                 unsigned long start, unsigned long end)
1546 {
1547         struct vm_area_struct *vma;
1548         int err = -EINVAL; /* in case end < start */
1549
1550         while (start < end) {
1551                 unsigned long local_start;
1552                 unsigned long local_end;
1553
1554                 vma = find_vma_intersection(mm, start, end);
1555                 if (!vma)
1556                         break;
1557
1558                 /* is it ours? */
1559                 if (vma->vm_ops != vm_ops)
1560                         goto try_next;
1561
1562                 local_start = vma->vm_start;
1563
1564                 if (start > local_start)
1565                         local_start = start;
1566
1567                 local_end = vma->vm_end;
1568
1569                 if (end < local_end)
1570                         local_end = end;
1571
1572                 err = zap_vma_ptes(vma, local_start, local_end - local_start);
1573                 if (unlikely(err))
1574                         break;
1575
1576 try_next:
1577                 /* go to next vma, if any */
1578                 start = vma->vm_end;
1579         }
1580
1581         return err;
1582 }
1583
1584 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
1585                 struct kbase_va_region *reg,
1586                 u64 new_pages, u64 old_pages)
1587 {
1588         phys_addr_t *phy_pages;
1589         u64 delta = new_pages - old_pages;
1590         int ret = 0;
1591
1592         lockdep_assert_held(&kctx->reg_lock);
1593
1594         /* Map the new pages into the GPU */
1595         phy_pages = kbase_get_gpu_phy_pages(reg);
1596         ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages,
1597                         phy_pages + old_pages, delta, reg->flags);
1598
1599         return ret;
1600 }
1601
1602 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
1603                 struct kbase_va_region *reg,
1604                 u64 new_pages, u64 old_pages)
1605 {
1606         struct kbase_mem_phy_alloc *cpu_alloc = reg->cpu_alloc;
1607         struct kbase_cpu_mapping *mapping;
1608         int err;
1609
1610         lockdep_assert_held(&kctx->process_mm->mmap_sem);
1611
1612         list_for_each_entry(mapping, &cpu_alloc->mappings, mappings_list) {
1613                 unsigned long mapping_size;
1614
1615                 mapping_size = (mapping->vm_end - mapping->vm_start)
1616                                 >> PAGE_SHIFT;
1617
1618                 /* is this mapping affected ?*/
1619                 if ((mapping->page_off + mapping_size) > new_pages) {
1620                         unsigned long first_bad = 0;
1621
1622                         if (new_pages > mapping->page_off)
1623                                 first_bad = new_pages - mapping->page_off;
1624
1625                         err = zap_range_nolock(current->mm,
1626                                         &kbase_vm_ops,
1627                                         mapping->vm_start +
1628                                         (first_bad << PAGE_SHIFT),
1629                                         mapping->vm_end);
1630
1631                         WARN(err,
1632                              "Failed to zap VA range (0x%lx - 0x%lx);\n",
1633                              mapping->vm_start +
1634                              (first_bad << PAGE_SHIFT),
1635                              mapping->vm_end
1636                              );
1637
1638                         /* The zap failed, give up and exit */
1639                         if (err)
1640                                 goto failed;
1641                 }
1642         }
1643
1644         return 0;
1645
1646 failed:
1647         return err;
1648 }
1649
1650 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
1651                 struct kbase_va_region *reg,
1652                 u64 new_pages, u64 old_pages)
1653 {
1654         u64 delta = old_pages - new_pages;
1655         int ret = 0;
1656
1657         ret = kbase_mmu_teardown_pages(kctx,
1658                         reg->start_pfn + new_pages, delta);
1659
1660         return ret;
1661 }
1662
1663 int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
1664 {
1665         u64 old_pages;
1666         u64 delta;
1667         int res = -EINVAL;
1668         struct kbase_va_region *reg;
1669         bool read_locked = false;
1670
1671         KBASE_DEBUG_ASSERT(kctx);
1672         KBASE_DEBUG_ASSERT(failure_reason);
1673         KBASE_DEBUG_ASSERT(gpu_addr != 0);
1674
1675         down_write(&current->mm->mmap_sem);
1676         kbase_gpu_vm_lock(kctx);
1677
1678         /* Validate the region */
1679         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1680         if (!reg || (reg->flags & KBASE_REG_FREE)) {
1681                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1682                 goto out_unlock;
1683         }
1684
1685         KBASE_DEBUG_ASSERT(reg->cpu_alloc);
1686         KBASE_DEBUG_ASSERT(reg->gpu_alloc);
1687
1688         if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
1689                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1690                 goto out_unlock;
1691         }
1692
1693         if (0 == (reg->flags & KBASE_REG_GROWABLE)) {
1694                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1695                 goto out_unlock;
1696         }
1697
1698         if (new_pages > reg->nr_pages) {
1699                 /* Would overflow the VA region */
1700                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1701                 goto out_unlock;
1702         }
1703
1704         /* can't be mapped more than once on the GPU */
1705         if (atomic_read(&reg->gpu_alloc->gpu_mappings) > 1) {
1706                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1707                 goto out_unlock;
1708         }
1709         /* can't grow regions which are ephemeral */
1710         if (reg->flags & KBASE_REG_DONT_NEED) {
1711                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1712                 goto out_unlock;
1713         }
1714
1715         if (new_pages == reg->gpu_alloc->nents) {
1716                 /* no change */
1717                 res = 0;
1718                 goto out_unlock;
1719         }
1720
1721         old_pages = kbase_reg_current_backed_size(reg);
1722         if (new_pages > old_pages) {
1723                 delta = new_pages - old_pages;
1724
1725                 /*
1726                  * No update to the mm so downgrade the writer lock to a read
1727                  * lock so other readers aren't blocked after this point.
1728                  */
1729                 downgrade_write(&current->mm->mmap_sem);
1730                 read_locked = true;
1731
1732                 /* Allocate some more pages */
1733                 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
1734                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1735                         goto out_unlock;
1736                 }
1737                 if (reg->cpu_alloc != reg->gpu_alloc) {
1738                         if (kbase_alloc_phy_pages_helper(
1739                                         reg->gpu_alloc, delta) != 0) {
1740                                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1741                                 kbase_free_phy_pages_helper(reg->cpu_alloc,
1742                                                 delta);
1743                                 goto out_unlock;
1744                         }
1745                 }
1746
1747                 /* No update required for CPU mappings, that's done on fault. */
1748
1749                 /* Update GPU mapping. */
1750                 res = kbase_mem_grow_gpu_mapping(kctx, reg,
1751                                 new_pages, old_pages);
1752
1753                 /* On error free the new pages */
1754                 if (res) {
1755                         kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1756                         if (reg->cpu_alloc != reg->gpu_alloc)
1757                                 kbase_free_phy_pages_helper(reg->gpu_alloc,
1758                                                 delta);
1759                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1760                         goto out_unlock;
1761                 }
1762         } else {
1763                 delta = old_pages - new_pages;
1764
1765                 /* Update all CPU mapping(s) */
1766                 res = kbase_mem_shrink_cpu_mapping(kctx, reg,
1767                                 new_pages, old_pages);
1768                 if (res) {
1769                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1770                         goto out_unlock;
1771                 }
1772
1773                 /* Update the GPU mapping */
1774                 res = kbase_mem_shrink_gpu_mapping(kctx, reg,
1775                                 new_pages, old_pages);
1776                 if (res) {
1777                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1778                         goto out_unlock;
1779                 }
1780
1781                 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1782                 if (reg->cpu_alloc != reg->gpu_alloc)
1783                         kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
1784         }
1785
1786 out_unlock:
1787         kbase_gpu_vm_unlock(kctx);
1788         if (read_locked)
1789                 up_read(&current->mm->mmap_sem);
1790         else
1791                 up_write(&current->mm->mmap_sem);
1792
1793         return res;
1794 }
1795
1796 static void kbase_cpu_vm_open(struct vm_area_struct *vma)
1797 {
1798         struct kbase_cpu_mapping *map = vma->vm_private_data;
1799
1800         KBASE_DEBUG_ASSERT(map);
1801         KBASE_DEBUG_ASSERT(map->count > 0);
1802         /* non-atomic as we're under Linux' mm lock */
1803         map->count++;
1804 }
1805
1806 static void kbase_cpu_vm_close(struct vm_area_struct *vma)
1807 {
1808         struct kbase_cpu_mapping *map = vma->vm_private_data;
1809
1810         KBASE_DEBUG_ASSERT(map);
1811         KBASE_DEBUG_ASSERT(map->count > 0);
1812
1813         /* non-atomic as we're under Linux' mm lock */
1814         if (--map->count)
1815                 return;
1816
1817         KBASE_DEBUG_ASSERT(map->kctx);
1818         KBASE_DEBUG_ASSERT(map->alloc);
1819
1820         kbase_gpu_vm_lock(map->kctx);
1821
1822         if (map->region) {
1823                 KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) ==
1824                                 KBASE_REG_ZONE_SAME_VA);
1825                 /* Avoid freeing memory on the process death which results in
1826                  * GPU Page Fault. Memory will be freed in kbase_destroy_context
1827                  */
1828                 if (!(current->flags & PF_EXITING))
1829                         kbase_mem_free_region(map->kctx, map->region);
1830         }
1831
1832         list_del(&map->mappings_list);
1833
1834         kbase_gpu_vm_unlock(map->kctx);
1835
1836         kbase_mem_phy_alloc_put(map->alloc);
1837         kfree(map);
1838 }
1839
1840 KBASE_EXPORT_TEST_API(kbase_cpu_vm_close);
1841
1842
1843 static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1844 {
1845         struct kbase_cpu_mapping *map = vma->vm_private_data;
1846         pgoff_t rel_pgoff;
1847         size_t i;
1848
1849         KBASE_DEBUG_ASSERT(map);
1850         KBASE_DEBUG_ASSERT(map->count > 0);
1851         KBASE_DEBUG_ASSERT(map->kctx);
1852         KBASE_DEBUG_ASSERT(map->alloc);
1853
1854         /* we don't use vmf->pgoff as it's affected by our mmap with
1855          * offset being a GPU VA or a cookie */
1856         rel_pgoff = ((unsigned long)vmf->virtual_address - map->vm_start)
1857                         >> PAGE_SHIFT;
1858
1859         kbase_gpu_vm_lock(map->kctx);
1860         if (map->page_off + rel_pgoff >= map->alloc->nents)
1861                 goto locked_bad_fault;
1862
1863         /* Fault on access to DONT_NEED regions */
1864         if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED))
1865                 goto locked_bad_fault;
1866
1867         /* insert all valid pages from the fault location */
1868         for (i = rel_pgoff;
1869              i < MIN((vma->vm_end - vma->vm_start) >> PAGE_SHIFT,
1870              map->alloc->nents - map->page_off); i++) {
1871                 int ret = vm_insert_pfn(vma, map->vm_start + (i << PAGE_SHIFT),
1872                     PFN_DOWN(map->alloc->pages[map->page_off + i]));
1873                 if (ret < 0 && ret != -EBUSY)
1874                         goto locked_bad_fault;
1875         }
1876
1877         kbase_gpu_vm_unlock(map->kctx);
1878         /* we resolved it, nothing for VM to do */
1879         return VM_FAULT_NOPAGE;
1880
1881 locked_bad_fault:
1882         kbase_gpu_vm_unlock(map->kctx);
1883         return VM_FAULT_SIGBUS;
1884 }
1885
1886 static const struct vm_operations_struct kbase_vm_ops = {
1887         .open  = kbase_cpu_vm_open,
1888         .close = kbase_cpu_vm_close,
1889         .fault = kbase_cpu_vm_fault
1890 };
1891
1892 static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close)
1893 {
1894         struct kbase_cpu_mapping *map;
1895         u64 start_off = vma->vm_pgoff - reg->start_pfn;
1896         phys_addr_t *page_array;
1897         int err = 0;
1898         int i;
1899
1900         map = kzalloc(sizeof(*map), GFP_KERNEL);
1901
1902         if (!map) {
1903                 WARN_ON(1);
1904                 err = -ENOMEM;
1905                 goto out;
1906         }
1907
1908         /*
1909          * VM_DONTCOPY - don't make this mapping available in fork'ed processes
1910          * VM_DONTEXPAND - disable mremap on this region
1911          * VM_IO - disables paging
1912          * VM_DONTDUMP - Don't include in core dumps (3.7 only)
1913          * VM_MIXEDMAP - Support mixing struct page*s and raw pfns.
1914          *               This is needed to support using the dedicated and
1915          *               the OS based memory backends together.
1916          */
1917         /*
1918          * This will need updating to propagate coherency flags
1919          * See MIDBASE-1057
1920          */
1921
1922 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
1923         vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO;
1924 #else
1925         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
1926 #endif
1927         vma->vm_ops = &kbase_vm_ops;
1928         vma->vm_private_data = map;
1929
1930         page_array = kbase_get_cpu_phy_pages(reg);
1931
1932         if (!(reg->flags & KBASE_REG_CPU_CACHED) &&
1933             (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) {
1934                 /* We can't map vmalloc'd memory uncached.
1935                  * Other memory will have been returned from
1936                  * kbase_mem_pool which would be
1937                  * suitable for mapping uncached.
1938                  */
1939                 BUG_ON(kaddr);
1940                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1941         }
1942
1943         if (!kaddr) {
1944                 unsigned long addr = vma->vm_start + aligned_offset;
1945
1946                 vma->vm_flags |= VM_PFNMAP;
1947                 for (i = 0; i < nr_pages; i++) {
1948                         unsigned long pfn = PFN_DOWN(page_array[i + start_off]);
1949
1950                         err = vm_insert_pfn(vma, addr, pfn);
1951                         if (WARN_ON(err))
1952                                 break;
1953
1954                         addr += PAGE_SIZE;
1955                 }
1956         } else {
1957                 WARN_ON(aligned_offset);
1958                 /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */
1959                 vma->vm_flags |= VM_MIXEDMAP;
1960                 /* vmalloc remaping is easy... */
1961                 err = remap_vmalloc_range(vma, kaddr, 0);
1962                 WARN_ON(err);
1963         }
1964
1965         if (err) {
1966                 kfree(map);
1967                 goto out;
1968         }
1969
1970         map->page_off = start_off;
1971         map->region = free_on_close ? reg : NULL;
1972         map->kctx = reg->kctx;
1973         map->vm_start = vma->vm_start + aligned_offset;
1974         if (aligned_offset) {
1975                 KBASE_DEBUG_ASSERT(!start_off);
1976                 map->vm_end = map->vm_start + (reg->nr_pages << PAGE_SHIFT);
1977         } else {
1978                 map->vm_end = vma->vm_end;
1979         }
1980         map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
1981         map->count = 1; /* start with one ref */
1982
1983         if (reg->flags & KBASE_REG_CPU_CACHED)
1984                 map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
1985
1986         list_add(&map->mappings_list, &map->alloc->mappings);
1987
1988  out:
1989         return err;
1990 }
1991
1992 static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr)
1993 {
1994         struct kbase_va_region *new_reg;
1995         u32 nr_pages;
1996         size_t size;
1997         int err = 0;
1998         u32 *tb;
1999         int owns_tb = 1;
2000
2001         dev_dbg(kctx->kbdev->dev, "in %s\n", __func__);
2002         size = (vma->vm_end - vma->vm_start);
2003         nr_pages = size >> PAGE_SHIFT;
2004
2005         if (!kctx->jctx.tb) {
2006                 KBASE_DEBUG_ASSERT(0 != size);
2007                 tb = vmalloc_user(size);
2008
2009                 if (NULL == tb) {
2010                         err = -ENOMEM;
2011                         goto out;
2012                 }
2013
2014                 err = kbase_device_trace_buffer_install(kctx, tb, size);
2015                 if (err) {
2016                         vfree(tb);
2017                         goto out;
2018                 }
2019         } else {
2020                 err = -EINVAL;
2021                 goto out;
2022         }
2023
2024         *kaddr = kctx->jctx.tb;
2025
2026         new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2027         if (!new_reg) {
2028                 err = -ENOMEM;
2029                 WARN_ON(1);
2030                 goto out_no_region;
2031         }
2032
2033         new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
2034         if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2035                 err = -ENOMEM;
2036                 new_reg->cpu_alloc = NULL;
2037                 WARN_ON(1);
2038                 goto out_no_alloc;
2039         }
2040
2041         new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2042
2043         new_reg->cpu_alloc->imported.kctx = kctx;
2044         new_reg->flags &= ~KBASE_REG_FREE;
2045         new_reg->flags |= KBASE_REG_CPU_CACHED;
2046
2047         /* alloc now owns the tb */
2048         owns_tb = 0;
2049
2050         if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2051                 err = -ENOMEM;
2052                 WARN_ON(1);
2053                 goto out_no_va_region;
2054         }
2055
2056         *reg = new_reg;
2057
2058         /* map read only, noexec */
2059         vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2060         /* the rest of the flags is added by the cpu_mmap handler */
2061
2062         dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
2063         return 0;
2064
2065 out_no_va_region:
2066 out_no_alloc:
2067         kbase_free_alloced_region(new_reg);
2068 out_no_region:
2069         if (owns_tb) {
2070                 kbase_device_trace_buffer_uninstall(kctx);
2071                 vfree(tb);
2072         }
2073 out:
2074         return err;
2075 }
2076
2077 static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr)
2078 {
2079         struct kbase_va_region *new_reg;
2080         void *kaddr;
2081         u32 nr_pages;
2082         size_t size;
2083         int err = 0;
2084
2085         dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
2086         size = (vma->vm_end - vma->vm_start);
2087         nr_pages = size >> PAGE_SHIFT;
2088
2089         kaddr = kbase_mmu_dump(kctx, nr_pages);
2090
2091         if (!kaddr) {
2092                 err = -ENOMEM;
2093                 goto out;
2094         }
2095
2096         new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2097         if (!new_reg) {
2098                 err = -ENOMEM;
2099                 WARN_ON(1);
2100                 goto out;
2101         }
2102
2103         new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
2104         if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2105                 err = -ENOMEM;
2106                 new_reg->cpu_alloc = NULL;
2107                 WARN_ON(1);
2108                 goto out_no_alloc;
2109         }
2110
2111         new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2112
2113         new_reg->flags &= ~KBASE_REG_FREE;
2114         new_reg->flags |= KBASE_REG_CPU_CACHED;
2115         if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2116                 err = -ENOMEM;
2117                 WARN_ON(1);
2118                 goto out_va_region;
2119         }
2120
2121         *kmap_addr = kaddr;
2122         *reg = new_reg;
2123
2124         dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
2125         return 0;
2126
2127 out_no_alloc:
2128 out_va_region:
2129         kbase_free_alloced_region(new_reg);
2130 out:
2131         return err;
2132 }
2133
2134
2135 void kbase_os_mem_map_lock(struct kbase_context *kctx)
2136 {
2137         struct mm_struct *mm = current->mm;
2138         (void)kctx;
2139         down_read(&mm->mmap_sem);
2140 }
2141
2142 void kbase_os_mem_map_unlock(struct kbase_context *kctx)
2143 {
2144         struct mm_struct *mm = current->mm;
2145         (void)kctx;
2146         up_read(&mm->mmap_sem);
2147 }
2148
2149 #if defined(CONFIG_DMA_SHARED_BUFFER) && defined(CONFIG_MALI_TRACE_TIMELINE)
2150 /* This section is required only for instrumentation. */
2151
2152 static void kbase_dma_buf_vm_open(struct vm_area_struct *vma)
2153 {
2154         struct kbase_cpu_mapping *map = vma->vm_private_data;
2155
2156         KBASE_DEBUG_ASSERT(map);
2157         KBASE_DEBUG_ASSERT(map->count > 0);
2158         /* Non-atomic as we're under Linux's mm lock. */
2159         map->count++;
2160 }
2161
2162 static void kbase_dma_buf_vm_close(struct vm_area_struct *vma)
2163 {
2164         struct kbase_cpu_mapping *map = vma->vm_private_data;
2165
2166         KBASE_DEBUG_ASSERT(map);
2167         KBASE_DEBUG_ASSERT(map->count > 0);
2168
2169         /* Non-atomic as we're under Linux's mm lock. */
2170         if (--map->count)
2171                 return;
2172
2173         KBASE_DEBUG_ASSERT(map->kctx);
2174
2175         kbase_gpu_vm_lock(map->kctx);
2176         list_del(&map->mappings_list);
2177         kbase_gpu_vm_unlock(map->kctx);
2178         kfree(map);
2179 }
2180
2181 static const struct vm_operations_struct kbase_dma_mmap_ops = {
2182         .open  = kbase_dma_buf_vm_open,
2183         .close = kbase_dma_buf_vm_close,
2184 };
2185 #endif /* CONFIG_DMA_SHARED_BUFFER && CONFIG_MALI_TRACE_TIMELINE */
2186
2187 int kbase_mmap(struct file *file, struct vm_area_struct *vma)
2188 {
2189         struct kbase_context *kctx = file->private_data;
2190         struct kbase_va_region *reg;
2191         void *kaddr = NULL;
2192         size_t nr_pages;
2193         int err = 0;
2194         int free_on_close = 0;
2195         struct device *dev = kctx->kbdev->dev;
2196         size_t aligned_offset = 0;
2197
2198         dev_dbg(dev, "kbase_mmap\n");
2199         nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2200
2201         /* strip away corresponding VM_MAY% flags to the VM_% flags requested */
2202         vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4);
2203
2204         if (0 == nr_pages) {
2205                 err = -EINVAL;
2206                 goto out;
2207         }
2208
2209         if (!(vma->vm_flags & VM_SHARED)) {
2210                 err = -EINVAL;
2211                 goto out;
2212         }
2213
2214         kbase_gpu_vm_lock(kctx);
2215
2216         if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) {
2217                 /* The non-mapped tracking helper page */
2218                 err = kbase_tracking_page_setup(kctx, vma);
2219                 goto out_unlock;
2220         }
2221
2222         /* if not the MTP, verify that the MTP has been mapped */
2223         rcu_read_lock();
2224         /* catches both when the special page isn't present or
2225          * when we've forked */
2226         if (rcu_dereference(kctx->process_mm) != current->mm) {
2227                 err = -EINVAL;
2228                 rcu_read_unlock();
2229                 goto out_unlock;
2230         }
2231         rcu_read_unlock();
2232
2233         switch (vma->vm_pgoff) {
2234         case PFN_DOWN(BASEP_MEM_INVALID_HANDLE):
2235         case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE):
2236                 /* Illegal handle for direct map */
2237                 err = -EINVAL;
2238                 goto out_unlock;
2239         case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE):
2240                 err = kbase_trace_buffer_mmap(kctx, vma, &reg, &kaddr);
2241                 if (0 != err)
2242                         goto out_unlock;
2243                 dev_dbg(dev, "kbase_trace_buffer_mmap ok\n");
2244                 /* free the region on munmap */
2245                 free_on_close = 1;
2246                 goto map;
2247         case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE):
2248                 /* MMU dump */
2249                 err = kbase_mmu_dump_mmap(kctx, vma, &reg, &kaddr);
2250                 if (0 != err)
2251                         goto out_unlock;
2252                 /* free the region on munmap */
2253                 free_on_close = 1;
2254                 goto map;
2255         case PFN_DOWN(BASE_MEM_COOKIE_BASE) ...
2256              PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: {
2257                 /* SAME_VA stuff, fetch the right region */
2258                 int gpu_pc_bits;
2259                 int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
2260
2261                 gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
2262                 reg = kctx->pending_regions[cookie];
2263                 if (!reg) {
2264                         err = -ENOMEM;
2265                         goto out_unlock;
2266                 }
2267
2268                 if (reg->flags & KBASE_REG_ALIGNED) {
2269                         /* nr_pages must be able to hold alignment pages
2270                          * plus actual pages */
2271                         unsigned long align = 1ULL << gpu_pc_bits;
2272                         unsigned long extra_pages = 3 * PFN_DOWN(align);
2273                         unsigned long aligned_addr;
2274                         unsigned long aligned_addr_end;
2275                         unsigned long nr_bytes = reg->nr_pages << PAGE_SHIFT;
2276
2277                         if (kctx->api_version < KBASE_API_VERSION(8, 5))
2278                                 /* Maintain compatibility with old userspace */
2279                                 extra_pages = PFN_DOWN(align);
2280
2281                         if (nr_pages != reg->nr_pages + extra_pages) {
2282                                 /* incorrect mmap size */
2283                                 /* leave the cookie for a potential
2284                                  * later mapping, or to be reclaimed
2285                                  * later when the context is freed */
2286                                 err = -ENOMEM;
2287                                 goto out_unlock;
2288                         }
2289
2290                         aligned_addr = ALIGN(vma->vm_start, align);
2291                         aligned_addr_end = aligned_addr + nr_bytes;
2292
2293                         if (kctx->api_version >= KBASE_API_VERSION(8, 5)) {
2294                                 if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
2295                                         /* Can't end at 4GB boundary */
2296                                         aligned_addr += 2 * align;
2297                                 } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
2298                                         /* Can't start at 4GB boundary */
2299                                         aligned_addr += align;
2300                                 }
2301                         }
2302
2303                         aligned_offset = aligned_addr - vma->vm_start;
2304                 } else if (reg->nr_pages != nr_pages) {
2305                         /* incorrect mmap size */
2306                         /* leave the cookie for a potential later
2307                          * mapping, or to be reclaimed later when the
2308                          * context is freed */
2309                         err = -ENOMEM;
2310                         goto out_unlock;
2311                 }
2312
2313                 if ((vma->vm_flags & VM_READ &&
2314                                         !(reg->flags & KBASE_REG_CPU_RD)) ||
2315                                 (vma->vm_flags & VM_WRITE &&
2316                                  !(reg->flags & KBASE_REG_CPU_WR))) {
2317                         /* VM flags inconsistent with region flags */
2318                         err = -EPERM;
2319                         dev_err(dev, "%s:%d inconsistent VM flags\n",
2320                                         __FILE__, __LINE__);
2321                         goto out_unlock;
2322                 }
2323
2324                 /* adjust down nr_pages to what we have physically */
2325                 nr_pages = kbase_reg_current_backed_size(reg);
2326
2327                 if (kbase_gpu_mmap(kctx, reg,
2328                                         vma->vm_start + aligned_offset,
2329                                         reg->nr_pages, 1) != 0) {
2330                         dev_err(dev, "%s:%d\n", __FILE__, __LINE__);
2331                         /* Unable to map in GPU space. */
2332                         WARN_ON(1);
2333                         err = -ENOMEM;
2334                         goto out_unlock;
2335                 }
2336
2337                 /* no need for the cookie anymore */
2338                 kctx->pending_regions[cookie] = NULL;
2339                 kctx->cookies |= (1UL << cookie);
2340
2341                 /*
2342                  * Overwrite the offset with the
2343                  * region start_pfn, so we effectively
2344                  * map from offset 0 in the region.
2345                  */
2346                 vma->vm_pgoff = reg->start_pfn;
2347
2348                 /* free the region on munmap */
2349                 free_on_close = 1;
2350                 goto map;
2351         }
2352         default: {
2353                 reg = kbase_region_tracker_find_region_enclosing_address(kctx, (u64)vma->vm_pgoff << PAGE_SHIFT);
2354
2355                 if (reg && !(reg->flags & KBASE_REG_FREE)) {
2356                         /* will this mapping overflow the size of the region? */
2357                         if (nr_pages > (reg->nr_pages - (vma->vm_pgoff - reg->start_pfn)))
2358                                 goto overflow;
2359
2360                         if ((vma->vm_flags & VM_READ &&
2361                              !(reg->flags & KBASE_REG_CPU_RD)) ||
2362                             (vma->vm_flags & VM_WRITE &&
2363                              !(reg->flags & KBASE_REG_CPU_WR))) {
2364                                 /* VM flags inconsistent with region flags */
2365                                 err = -EPERM;
2366                                 dev_err(dev, "%s:%d inconsistent VM flags\n",
2367                                         __FILE__, __LINE__);
2368                                 goto out_unlock;
2369                         }
2370
2371 #ifdef CONFIG_DMA_SHARED_BUFFER
2372                         if (reg->cpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM)
2373                                 goto dma_map;
2374 #endif /* CONFIG_DMA_SHARED_BUFFER */
2375
2376                         /* limit what we map to the amount currently backed */
2377                         if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
2378                                 if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
2379                                         nr_pages = 0;
2380                                 else
2381                                         nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn);
2382                         }
2383
2384                         goto map;
2385                 }
2386
2387 overflow:
2388                 err = -ENOMEM;
2389                 goto out_unlock;
2390         } /* default */
2391         } /* switch */
2392 map:
2393         err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close);
2394
2395         if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) {
2396                 /* MMU dump - userspace should now have a reference on
2397                  * the pages, so we can now free the kernel mapping */
2398                 vfree(kaddr);
2399         }
2400         goto out_unlock;
2401
2402 #ifdef CONFIG_DMA_SHARED_BUFFER
2403 dma_map:
2404         err = dma_buf_mmap(reg->cpu_alloc->imported.umm.dma_buf, vma, vma->vm_pgoff - reg->start_pfn);
2405 #if defined(CONFIG_MALI_TRACE_TIMELINE)
2406         /* This section is required only for instrumentation. */
2407         /* Add created mapping to imported region mapping list.
2408          * It is important to make it visible to dumping infrastructure.
2409          * Add mapping only if vm_ops structure is not used by memory owner. */
2410         WARN_ON(vma->vm_ops);
2411         WARN_ON(vma->vm_private_data);
2412         if (!err && !vma->vm_ops && !vma->vm_private_data) {
2413                 struct kbase_cpu_mapping *map = kzalloc(
2414                         sizeof(*map),
2415                         GFP_KERNEL);
2416
2417                 if (map) {
2418                         map->kctx     = reg->kctx;
2419                         map->region   = NULL;
2420                         map->page_off = vma->vm_pgoff;
2421                         map->vm_start = vma->vm_start;
2422                         map->vm_end   = vma->vm_end;
2423                         map->count    = 1; /* start with one ref */
2424
2425                         vma->vm_ops          = &kbase_dma_mmap_ops;
2426                         vma->vm_private_data = map;
2427
2428                         list_add(
2429                                 &map->mappings_list,
2430                                 &reg->cpu_alloc->mappings);
2431                 }
2432         }
2433 #endif /* CONFIG_MALI_TRACE_TIMELINE */
2434 #endif /* CONFIG_DMA_SHARED_BUFFER */
2435 out_unlock:
2436         kbase_gpu_vm_unlock(kctx);
2437 out:
2438         if (err)
2439                 dev_err(dev, "mmap failed %d\n", err);
2440
2441         return err;
2442 }
2443
2444 KBASE_EXPORT_TEST_API(kbase_mmap);
2445
2446 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2447                       unsigned long prot_request, struct kbase_vmap_struct *map)
2448 {
2449         struct kbase_va_region *reg;
2450         unsigned long page_index;
2451         unsigned int offset = gpu_addr & ~PAGE_MASK;
2452         size_t page_count = PFN_UP(offset + size);
2453         phys_addr_t *page_array;
2454         struct page **pages;
2455         void *cpu_addr = NULL;
2456         pgprot_t prot;
2457         size_t i;
2458         bool sync_needed;
2459
2460         if (!size || !map)
2461                 return NULL;
2462
2463         /* check if page_count calculation will wrap */
2464         if (size > ((size_t)-1 / PAGE_SIZE))
2465                 return NULL;
2466
2467         kbase_gpu_vm_lock(kctx);
2468
2469         reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
2470         if (!reg || (reg->flags & KBASE_REG_FREE))
2471                 goto out_unlock;
2472
2473         page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn;
2474
2475         /* check if page_index + page_count will wrap */
2476         if (-1UL - page_count < page_index)
2477                 goto out_unlock;
2478
2479         if (page_index + page_count > kbase_reg_current_backed_size(reg))
2480                 goto out_unlock;
2481
2482         if (reg->flags & KBASE_REG_DONT_NEED)
2483                 goto out_unlock;
2484
2485         /* check access permissions can be satisfied
2486          * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */
2487         if ((reg->flags & prot_request) != prot_request)
2488                 goto out_unlock;
2489
2490         page_array = kbase_get_cpu_phy_pages(reg);
2491         if (!page_array)
2492                 goto out_unlock;
2493
2494         pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
2495         if (!pages)
2496                 goto out_unlock;
2497
2498         for (i = 0; i < page_count; i++)
2499                 pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i]));
2500
2501         prot = PAGE_KERNEL;
2502         if (!(reg->flags & KBASE_REG_CPU_CACHED)) {
2503                 /* Map uncached */
2504                 prot = pgprot_writecombine(prot);
2505         }
2506         /* Note: enforcing a RO prot_request onto prot is not done, since:
2507          * - CPU-arch-specific integration required
2508          * - kbase_vmap() requires no access checks to be made/enforced */
2509
2510         cpu_addr = vmap(pages, page_count, VM_MAP, prot);
2511
2512         kfree(pages);
2513
2514         if (!cpu_addr)
2515                 goto out_unlock;
2516
2517         map->gpu_addr = gpu_addr;
2518         map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2519         map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index];
2520         map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
2521         map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
2522         map->addr = (void *)((uintptr_t)cpu_addr + offset);
2523         map->size = size;
2524         map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0;
2525         sync_needed = map->is_cached;
2526
2527 #ifdef CONFIG_MALI_COH_KERN
2528         /* kernel can use coherent memory if supported */
2529         if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2530                 sync_needed = false;
2531 #endif
2532
2533         if (sync_needed) {
2534                 /* Sync first page */
2535                 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2536                 phys_addr_t cpu_pa = map->cpu_pages[0];
2537                 phys_addr_t gpu_pa = map->gpu_pages[0];
2538
2539                 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2540                                 KBASE_SYNC_TO_CPU);
2541
2542                 /* Sync middle pages (if any) */
2543                 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2544                         cpu_pa = map->cpu_pages[i];
2545                         gpu_pa = map->gpu_pages[i];
2546                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2547                                         KBASE_SYNC_TO_CPU);
2548                 }
2549
2550                 /* Sync last page (if any) */
2551                 if (page_count > 1) {
2552                         cpu_pa = map->cpu_pages[page_count - 1];
2553                         gpu_pa = map->gpu_pages[page_count - 1];
2554                         sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2555                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2556                                         KBASE_SYNC_TO_CPU);
2557                 }
2558         }
2559         kbase_gpu_vm_unlock(kctx);
2560
2561         return map->addr;
2562
2563 out_unlock:
2564         kbase_gpu_vm_unlock(kctx);
2565         return NULL;
2566 }
2567
2568 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2569                 struct kbase_vmap_struct *map)
2570 {
2571         /* 0 is specified for prot_request to indicate no access checks should
2572          * be made.
2573          *
2574          * As mentioned in kbase_vmap_prot() this means that a kernel-side
2575          * CPU-RO mapping is not enforced to allow this to work */
2576         return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map);
2577 }
2578 KBASE_EXPORT_TEST_API(kbase_vmap);
2579
2580 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
2581 {
2582         void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
2583         bool sync_needed = map->is_cached;
2584         vunmap(addr);
2585 #ifdef CONFIG_MALI_COH_KERN
2586         /* kernel can use coherent memory if supported */
2587         if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2588                 sync_needed = false;
2589 #endif
2590         if (sync_needed) {
2591                 off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
2592                 size_t size = map->size;
2593                 size_t page_count = PFN_UP(offset + size);
2594                 size_t i;
2595
2596                 /* Sync first page */
2597                 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2598                 phys_addr_t cpu_pa = map->cpu_pages[0];
2599                 phys_addr_t gpu_pa = map->gpu_pages[0];
2600
2601                 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2602                                 KBASE_SYNC_TO_DEVICE);
2603
2604                 /* Sync middle pages (if any) */
2605                 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2606                         cpu_pa = map->cpu_pages[i];
2607                         gpu_pa = map->gpu_pages[i];
2608                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2609                                         KBASE_SYNC_TO_DEVICE);
2610                 }
2611
2612                 /* Sync last page (if any) */
2613                 if (page_count > 1) {
2614                         cpu_pa = map->cpu_pages[page_count - 1];
2615                         gpu_pa = map->gpu_pages[page_count - 1];
2616                         sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2617                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2618                                         KBASE_SYNC_TO_DEVICE);
2619                 }
2620         }
2621         map->gpu_addr = 0;
2622         map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc);
2623         map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc);
2624         map->cpu_pages = NULL;
2625         map->gpu_pages = NULL;
2626         map->addr = NULL;
2627         map->size = 0;
2628         map->is_cached = false;
2629 }
2630 KBASE_EXPORT_TEST_API(kbase_vunmap);
2631
2632 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages)
2633 {
2634         struct mm_struct *mm;
2635
2636         rcu_read_lock();
2637         mm = rcu_dereference(kctx->process_mm);
2638         if (mm) {
2639                 atomic_add(pages, &kctx->nonmapped_pages);
2640 #ifdef SPLIT_RSS_COUNTING
2641                 add_mm_counter(mm, MM_FILEPAGES, pages);
2642 #else
2643                 spin_lock(&mm->page_table_lock);
2644                 add_mm_counter(mm, MM_FILEPAGES, pages);
2645                 spin_unlock(&mm->page_table_lock);
2646 #endif
2647         }
2648         rcu_read_unlock();
2649 }
2650
2651 static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx)
2652 {
2653         int pages;
2654         struct mm_struct *mm;
2655
2656         spin_lock(&kctx->mm_update_lock);
2657         mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock));
2658         if (!mm) {
2659                 spin_unlock(&kctx->mm_update_lock);
2660                 return;
2661         }
2662
2663         rcu_assign_pointer(kctx->process_mm, NULL);
2664         spin_unlock(&kctx->mm_update_lock);
2665         synchronize_rcu();
2666
2667         pages = atomic_xchg(&kctx->nonmapped_pages, 0);
2668 #ifdef SPLIT_RSS_COUNTING
2669         add_mm_counter(mm, MM_FILEPAGES, -pages);
2670 #else
2671         spin_lock(&mm->page_table_lock);
2672         add_mm_counter(mm, MM_FILEPAGES, -pages);
2673         spin_unlock(&mm->page_table_lock);
2674 #endif
2675 }
2676
2677 static void kbase_special_vm_close(struct vm_area_struct *vma)
2678 {
2679         struct kbase_context *kctx;
2680
2681         kctx = vma->vm_private_data;
2682         kbasep_os_process_page_usage_drain(kctx);
2683 }
2684
2685 static const struct vm_operations_struct kbase_vm_special_ops = {
2686         .close = kbase_special_vm_close,
2687 };
2688
2689 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma)
2690 {
2691         /* check that this is the only tracking page */
2692         spin_lock(&kctx->mm_update_lock);
2693         if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) {
2694                 spin_unlock(&kctx->mm_update_lock);
2695                 return -EFAULT;
2696         }
2697
2698         rcu_assign_pointer(kctx->process_mm, current->mm);
2699
2700         spin_unlock(&kctx->mm_update_lock);
2701
2702         /* no real access */
2703         vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2704 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
2705         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
2706 #else
2707         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
2708 #endif
2709         vma->vm_ops = &kbase_vm_special_ops;
2710         vma->vm_private_data = kctx;
2711
2712         return 0;
2713 }
2714 void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle)
2715 {
2716         int i;
2717         int res;
2718         void *va;
2719         dma_addr_t  dma_pa;
2720         struct kbase_va_region *reg;
2721         phys_addr_t *page_array;
2722 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2723         DEFINE_DMA_ATTRS(attrs);
2724 #endif
2725
2726         u32 pages = ((size - 1) >> PAGE_SHIFT) + 1;
2727         u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR |
2728                     BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR;
2729
2730         KBASE_DEBUG_ASSERT(kctx != NULL);
2731         KBASE_DEBUG_ASSERT(0 != size);
2732         KBASE_DEBUG_ASSERT(0 != pages);
2733
2734         if (size == 0)
2735                 goto err;
2736
2737         /* All the alloc calls return zeroed memory */
2738 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2739         dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2740         va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, &attrs);
2741 #else
2742         va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL);
2743 #endif
2744         if (!va)
2745                 goto err;
2746
2747         /* Store the state so we can free it later. */
2748         handle->cpu_va = va;
2749         handle->dma_pa = dma_pa;
2750         handle->size   = size;
2751
2752
2753         reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA);
2754         if (!reg)
2755                 goto no_reg;
2756
2757         reg->flags &= ~KBASE_REG_FREE;
2758         kbase_update_region_flags(kctx, reg, flags);
2759
2760         reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
2761         if (IS_ERR_OR_NULL(reg->cpu_alloc))
2762                 goto no_alloc;
2763
2764         reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2765
2766         page_array = kbase_get_cpu_phy_pages(reg);
2767
2768         for (i = 0; i < pages; i++)
2769                 page_array[i] = dma_pa + (i << PAGE_SHIFT);
2770
2771         reg->cpu_alloc->nents = pages;
2772
2773         kbase_gpu_vm_lock(kctx);
2774         res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1);
2775         kbase_gpu_vm_unlock(kctx);
2776         if (res)
2777                 goto no_mmap;
2778
2779         return va;
2780
2781 no_mmap:
2782         kbase_mem_phy_alloc_put(reg->cpu_alloc);
2783         kbase_mem_phy_alloc_put(reg->gpu_alloc);
2784 no_alloc:
2785         kfree(reg);
2786 no_reg:
2787 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2788         dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs);
2789 #else
2790         dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa);
2791 #endif
2792 err:
2793         return NULL;
2794 }
2795 KBASE_EXPORT_SYMBOL(kbase_va_alloc);
2796
2797 void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle)
2798 {
2799         struct kbase_va_region *reg;
2800         int err;
2801 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2802         DEFINE_DMA_ATTRS(attrs);
2803 #endif
2804
2805         KBASE_DEBUG_ASSERT(kctx != NULL);
2806         KBASE_DEBUG_ASSERT(handle->cpu_va != NULL);
2807
2808         kbase_gpu_vm_lock(kctx);
2809         reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va);
2810         KBASE_DEBUG_ASSERT(reg);
2811         err = kbase_gpu_munmap(kctx, reg);
2812         kbase_gpu_vm_unlock(kctx);
2813         KBASE_DEBUG_ASSERT(!err);
2814
2815         kbase_mem_phy_alloc_put(reg->cpu_alloc);
2816         kbase_mem_phy_alloc_put(reg->gpu_alloc);
2817         kfree(reg);
2818
2819 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2820         dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2821         dma_free_attrs(kctx->kbdev->dev, handle->size,
2822                         handle->cpu_va, handle->dma_pa, &attrs);
2823 #else
2824         dma_free_writecombine(kctx->kbdev->dev, handle->size,
2825                                 handle->cpu_va, handle->dma_pa);
2826 #endif
2827 }
2828 KBASE_EXPORT_SYMBOL(kbase_va_free);
2829