drivers/gpu/arm/midgard/mali_kbase_mem_linux.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_mem_linux.c
  22  * Base kernel memory APIs, Linux implementation.
  23  */
  24
  25 #include <linux/compat.h>
  26 #include <linux/kernel.h>
  27 #include <linux/bug.h>
  28 #include <linux/mm.h>
  29 #include <linux/mman.h>
  30 #include <linux/fs.h>
  31 #include <linux/version.h>
  32 #include <linux/dma-mapping.h>
  33 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \
  34         (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
  35 #include <linux/dma-attrs.h>
  36 #endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */
  37 #ifdef CONFIG_DMA_SHARED_BUFFER
  38 #include <linux/dma-buf.h>
  39 #endif                          /* defined(CONFIG_DMA_SHARED_BUFFER) */
  40 #include <linux/shrinker.h>
  41 #include <linux/cache.h>
  42
  43 #include <mali_kbase.h>
  44 #include <mali_kbase_mem_linux.h>
  45 #include <mali_kbase_config_defaults.h>
  46 #include <mali_kbase_hwaccess_time.h>
  47 #include <mali_kbase_tlstream.h>
  48
  49 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
  50 static const struct vm_operations_struct kbase_vm_ops;
  51
  52 /**
  53  * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation
  54  * @kctx:      Context the region belongs to
  55  * @reg:       The GPU region
  56  * @new_pages: The number of pages after the shrink
  57  * @old_pages: The number of pages before the shrink
  58  *
  59  * Return: 0 on success, -errno on error.
  60  *
  61  * Shrink (or completely remove) all CPU mappings which reference the shrunk
  62  * part of the allocation.
  63  *
  64  * Note: Caller must be holding the processes mmap_sem lock.
  65  */
  66 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
  67                 struct kbase_va_region *reg,
  68                 u64 new_pages, u64 old_pages);
  69
  70 /**
  71  * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
  72  * @kctx:      Context the region belongs to
  73  * @reg:       The GPU region or NULL if there isn't one
  74  * @new_pages: The number of pages after the shrink
  75  * @old_pages: The number of pages before the shrink
  76  *
  77  * Return: 0 on success, negative -errno on error
  78  *
  79  * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
  80  * itself is unmodified as we still need to reserve the VA, only the page tables
  81  * will be modified by this function.
  82  */
  83 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
  84                 struct kbase_va_region *reg,
  85                 u64 new_pages, u64 old_pages);
  86
  87 struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, u64 *gpu_va, u16 *va_alignment)
  88 {
  89         int zone;
  90         int gpu_pc_bits;
  91         int cpu_va_bits;
  92         struct kbase_va_region *reg;
  93         struct device *dev;
  94
  95         KBASE_DEBUG_ASSERT(kctx);
  96         KBASE_DEBUG_ASSERT(flags);
  97         KBASE_DEBUG_ASSERT(gpu_va);
  98         KBASE_DEBUG_ASSERT(va_alignment);
  99
 100         dev = kctx->kbdev->dev;
 101         *va_alignment = 0; /* no alignment by default */
 102         *gpu_va = 0; /* return 0 on failure */
 103
 104         gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
 105         cpu_va_bits = BITS_PER_LONG;
 106
 107         if (0 == va_pages) {
 108                 dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!");
 109                 goto bad_size;
 110         }
 111
 112         if (va_pages > (U64_MAX / PAGE_SIZE))
 113                 /* 64-bit address range is the max */
 114                 goto bad_size;
 115
 116 #if defined(CONFIG_64BIT)
 117         if (kbase_ctx_flag(kctx, KCTX_COMPAT))
 118                 cpu_va_bits = 32;
 119 #endif
 120
 121         if (!kbase_check_alloc_flags(*flags)) {
 122                 dev_warn(dev,
 123                                 "kbase_mem_alloc called with bad flags (%llx)",
 124                                 (unsigned long long)*flags);
 125                 goto bad_flags;
 126         }
 127
 128         if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
 129                         !kbase_device_is_cpu_coherent(kctx->kbdev)) {
 130                 dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
 131                 goto bad_flags;
 132         }
 133         if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
 134                         !kbase_device_is_cpu_coherent(kctx->kbdev)) {
 135                 /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
 136                 *flags &= ~BASE_MEM_COHERENT_SYSTEM;
 137         }
 138
 139         /* Limit GPU executable allocs to GPU PC size */
 140         if ((*flags & BASE_MEM_PROT_GPU_EX) &&
 141             (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT)))
 142                 goto bad_ex_size;
 143
 144         /* find out which VA zone to use */
 145         if (*flags & BASE_MEM_SAME_VA)
 146                 zone = KBASE_REG_ZONE_SAME_VA;
 147         else if (*flags & BASE_MEM_PROT_GPU_EX)
 148                 zone = KBASE_REG_ZONE_EXEC;
 149         else
 150                 zone = KBASE_REG_ZONE_CUSTOM_VA;
 151
 152         reg = kbase_alloc_free_region(kctx, 0, va_pages, zone);
 153         if (!reg) {
 154                 dev_err(dev, "Failed to allocate free region");
 155                 goto no_region;
 156         }
 157
 158         kbase_update_region_flags(kctx, reg, *flags);
 159
 160         if (kbase_reg_prepare_native(reg, kctx) != 0) {
 161                 dev_err(dev, "Failed to prepare region");
 162                 goto prepare_failed;
 163         }
 164
 165         if (*flags & BASE_MEM_GROW_ON_GPF)
 166                 reg->extent = extent;
 167         else
 168                 reg->extent = 0;
 169
 170         if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) {
 171                 dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
 172                                 (unsigned long long)commit_pages,
 173                                 (unsigned long long)va_pages);
 174                 goto no_mem;
 175         }
 176
 177         kbase_gpu_vm_lock(kctx);
 178
 179         /* mmap needed to setup VA? */
 180         if (*flags & BASE_MEM_SAME_VA) {
 181                 unsigned long prot = PROT_NONE;
 182                 unsigned long va_size = va_pages << PAGE_SHIFT;
 183                 unsigned long va_map = va_size;
 184                 unsigned long cookie, cookie_nr;
 185                 unsigned long cpu_addr;
 186
 187                 /* Bind to a cookie */
 188                 if (!kctx->cookies) {
 189                         dev_err(dev, "No cookies available for allocation!");
 190                         kbase_gpu_vm_unlock(kctx);
 191                         goto no_cookie;
 192                 }
 193                 /* return a cookie */
 194                 cookie_nr = __ffs(kctx->cookies);
 195                 kctx->cookies &= ~(1UL << cookie_nr);
 196                 BUG_ON(kctx->pending_regions[cookie_nr]);
 197                 kctx->pending_regions[cookie_nr] = reg;
 198
 199                 kbase_gpu_vm_unlock(kctx);
 200
 201                 /* relocate to correct base */
 202                 cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE);
 203                 cookie <<= PAGE_SHIFT;
 204
 205                 /* See if we must align memory due to GPU PC bits vs CPU VA */
 206                 if ((*flags & BASE_MEM_PROT_GPU_EX) &&
 207                     (cpu_va_bits > gpu_pc_bits)) {
 208                         *va_alignment = gpu_pc_bits;
 209                         reg->flags |= KBASE_REG_ALIGNED;
 210                 }
 211
 212                 /*
 213                  * 10.1-10.4 UKU userland relies on the kernel to call mmap.
 214                  * For all other versions we can just return the cookie
 215                  */
 216                 if (kctx->api_version < KBASE_API_VERSION(10, 1) ||
 217                     kctx->api_version > KBASE_API_VERSION(10, 4)) {
 218                         *gpu_va = (u64) cookie;
 219                         return reg;
 220                 }
 221
 222                 /*
 223                  * To achieve alignment and avoid allocating on large alignment
 224                  * (to work around a GPU hardware issue) we must allocate 3
 225                  * times the required size.
 226                  */
 227                 if (*va_alignment)
 228                         va_map += 3 * (1UL << *va_alignment);
 229
 230                 if (*flags & BASE_MEM_PROT_CPU_RD)
 231                         prot |= PROT_READ;
 232                 if (*flags & BASE_MEM_PROT_CPU_WR)
 233                         prot |= PROT_WRITE;
 234
 235                 cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot,
 236                                 MAP_SHARED, cookie);
 237
 238                 if (IS_ERR_VALUE(cpu_addr)) {
 239                         kbase_gpu_vm_lock(kctx);
 240                         kctx->pending_regions[cookie_nr] = NULL;
 241                         kctx->cookies |= (1UL << cookie_nr);
 242                         kbase_gpu_vm_unlock(kctx);
 243                         goto no_mmap;
 244                 }
 245
 246                 /*
 247                  * If we had to allocate extra VA space to force the
 248                  * alignment release it.
 249                  */
 250                 if (*va_alignment) {
 251                         unsigned long alignment = 1UL << *va_alignment;
 252                         unsigned long align_mask = alignment - 1;
 253                         unsigned long addr;
 254                         unsigned long addr_end;
 255                         unsigned long aligned_addr;
 256                         unsigned long aligned_addr_end;
 257
 258                         addr = cpu_addr;
 259                         addr_end = addr + va_map;
 260
 261                         aligned_addr = (addr + align_mask) &
 262                                         ~((u64) align_mask);
 263                         aligned_addr_end = aligned_addr + va_size;
 264
 265                         if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
 266                                 /*
 267                                  * Can't end at 4GB boundary on some GPUs as
 268                                  * it will halt the shader.
 269                                  */
 270                                 aligned_addr += 2 * alignment;
 271                                 aligned_addr_end += 2 * alignment;
 272                         } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
 273                                 /*
 274                                  * Can't start at 4GB boundary on some GPUs as
 275                                  * it will halt the shader.
 276                                  */
 277                                 aligned_addr += alignment;
 278                                 aligned_addr_end += alignment;
 279                         }
 280
 281                         /* anything to chop off at the start? */
 282                         if (addr != aligned_addr)
 283                                 vm_munmap(addr, aligned_addr - addr);
 284
 285                         /* anything at the end? */
 286                         if (addr_end != aligned_addr_end)
 287                                 vm_munmap(aligned_addr_end,
 288                                                 addr_end - aligned_addr_end);
 289
 290                         *gpu_va = (u64) aligned_addr;
 291                 } else
 292                         *gpu_va = (u64) cpu_addr;
 293         } else /* we control the VA */ {
 294                 if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) {
 295                         dev_warn(dev, "Failed to map memory on GPU");
 296                         kbase_gpu_vm_unlock(kctx);
 297                         goto no_mmap;
 298                 }
 299                 /* return real GPU VA */
 300                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
 301
 302                 kbase_gpu_vm_unlock(kctx);
 303         }
 304
 305         return reg;
 306
 307 no_mmap:
 308 no_cookie:
 309 no_mem:
 310         kbase_mem_phy_alloc_put(reg->cpu_alloc);
 311         kbase_mem_phy_alloc_put(reg->gpu_alloc);
 312 prepare_failed:
 313         kfree(reg);
 314 no_region:
 315 bad_ex_size:
 316 bad_flags:
 317 bad_size:
 318         return NULL;
 319 }
 320 KBASE_EXPORT_TEST_API(kbase_mem_alloc);
 321
 322 int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out)
 323 {
 324         struct kbase_va_region *reg;
 325         int ret = -EINVAL;
 326
 327         KBASE_DEBUG_ASSERT(kctx);
 328         KBASE_DEBUG_ASSERT(out);
 329
 330         kbase_gpu_vm_lock(kctx);
 331
 332         /* Validate the region */
 333         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
 334         if (!reg || (reg->flags & KBASE_REG_FREE))
 335                 goto out_unlock;
 336
 337         switch (query) {
 338         case KBASE_MEM_QUERY_COMMIT_SIZE:
 339                 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) {
 340                         *out = kbase_reg_current_backed_size(reg);
 341                 } else {
 342                         size_t i;
 343                         struct kbase_aliased *aliased;
 344                         *out = 0;
 345                         aliased = reg->cpu_alloc->imported.alias.aliased;
 346                         for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++)
 347                                 *out += aliased[i].length;
 348                 }
 349                 break;
 350         case KBASE_MEM_QUERY_VA_SIZE:
 351                 *out = reg->nr_pages;
 352                 break;
 353         case KBASE_MEM_QUERY_FLAGS:
 354         {
 355                 *out = 0;
 356                 if (KBASE_REG_CPU_WR & reg->flags)
 357                         *out |= BASE_MEM_PROT_CPU_WR;
 358                 if (KBASE_REG_CPU_RD & reg->flags)
 359                         *out |= BASE_MEM_PROT_CPU_RD;
 360                 if (KBASE_REG_CPU_CACHED & reg->flags)
 361                         *out |= BASE_MEM_CACHED_CPU;
 362                 if (KBASE_REG_GPU_WR & reg->flags)
 363                         *out |= BASE_MEM_PROT_GPU_WR;
 364                 if (KBASE_REG_GPU_RD & reg->flags)
 365                         *out |= BASE_MEM_PROT_GPU_RD;
 366                 if (!(KBASE_REG_GPU_NX & reg->flags))
 367                         *out |= BASE_MEM_PROT_GPU_EX;
 368                 if (KBASE_REG_SHARE_BOTH & reg->flags)
 369                         *out |= BASE_MEM_COHERENT_SYSTEM;
 370                 if (KBASE_REG_SHARE_IN & reg->flags)
 371                         *out |= BASE_MEM_COHERENT_LOCAL;
 372                 break;
 373         }
 374         default:
 375                 *out = 0;
 376                 goto out_unlock;
 377         }
 378
 379         ret = 0;
 380
 381 out_unlock:
 382         kbase_gpu_vm_unlock(kctx);
 383         return ret;
 384 }
 385
 386 /**
 387  * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the
 388  * Ephemeral memory eviction list.
 389  * @s:        Shrinker
 390  * @sc:       Shrinker control
 391  *
 392  * Return: Number of pages which can be freed.
 393  */
 394 static
 395 unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s,
 396                 struct shrink_control *sc)
 397 {
 398         struct kbase_context *kctx;
 399         struct kbase_mem_phy_alloc *alloc;
 400         unsigned long pages = 0;
 401
 402         kctx = container_of(s, struct kbase_context, reclaim);
 403
 404         mutex_lock(&kctx->evict_lock);
 405
 406         list_for_each_entry(alloc, &kctx->evict_list, evict_node)
 407                 pages += alloc->nents;
 408
 409         mutex_unlock(&kctx->evict_lock);
 410         return pages;
 411 }
 412
 413 /**
 414  * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction
 415  * list for pages and try to reclaim them.
 416  * @s:        Shrinker
 417  * @sc:       Shrinker control
 418  *
 419  * Return: Number of pages freed (can be less then requested) or -1 if the
 420  * shrinker failed to free pages in its pool.
 421  *
 422  * Note:
 423  * This function accesses region structures without taking the region lock,
 424  * this is required as the OOM killer can call the shrinker after the region
 425  * lock has already been held.
 426  * This is safe as we can guarantee that a region on the eviction list will
 427  * not be freed (kbase_mem_free_region removes the allocation from the list
 428  * before destroying it), or modified by other parts of the driver.
 429  * The eviction list itself is guarded by the eviction lock and the MMU updates
 430  * are protected by their own lock.
 431  */
 432 static
 433 unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
 434                 struct shrink_control *sc)
 435 {
 436         struct kbase_context *kctx;
 437         struct kbase_mem_phy_alloc *alloc;
 438         struct kbase_mem_phy_alloc *tmp;
 439         unsigned long freed = 0;
 440
 441         kctx = container_of(s, struct kbase_context, reclaim);
 442         mutex_lock(&kctx->evict_lock);
 443
 444         list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
 445                 int err;
 446
 447                 err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
 448                                 0, alloc->nents);
 449                 if (err != 0) {
 450                         /*
 451                          * Failed to remove GPU mapping, tell the shrinker
 452                          * to stop trying to shrink our slab even though we
 453                          * have pages in it.
 454                          */
 455                         freed = -1;
 456                         goto out_unlock;
 457                 }
 458
 459                 /*
 460                  * Update alloc->evicted before freeing the backing so the
 461                  * helper can determine that it needs to bypass the accounting
 462                  * and memory pool.
 463                  */
 464                 alloc->evicted = alloc->nents;
 465
 466                 kbase_free_phy_pages_helper(alloc, alloc->evicted);
 467                 freed += alloc->evicted;
 468                 list_del_init(&alloc->evict_node);
 469
 470                 /*
 471                  * Inform the JIT allocator this region has lost backing
 472                  * as it might need to free the allocation.
 473                  */
 474                 kbase_jit_backing_lost(alloc->reg);
 475
 476                 /* Enough pages have been freed so stop now */
 477                 if (freed > sc->nr_to_scan)
 478                         break;
 479         }
 480 out_unlock:
 481         mutex_unlock(&kctx->evict_lock);
 482
 483         return freed;
 484 }
 485
 486 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 487 static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s,
 488                 struct shrink_control *sc)
 489 {
 490         if (sc->nr_to_scan == 0)
 491                 return kbase_mem_evictable_reclaim_count_objects(s, sc);
 492
 493         return kbase_mem_evictable_reclaim_scan_objects(s, sc);
 494 }
 495 #endif
 496
 497 int kbase_mem_evictable_init(struct kbase_context *kctx)
 498 {
 499         INIT_LIST_HEAD(&kctx->evict_list);
 500         mutex_init(&kctx->evict_lock);
 501
 502         /* Register shrinker */
 503 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 504         kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink;
 505 #else
 506         kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects;
 507         kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects;
 508 #endif
 509         kctx->reclaim.seeks = DEFAULT_SEEKS;
 510         /* Kernel versions prior to 3.1 :
 511          * struct shrinker does not define batch */
 512 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
 513         kctx->reclaim.batch = 0;
 514 #endif
 515         register_shrinker(&kctx->reclaim);
 516         return 0;
 517 }
 518
 519 void kbase_mem_evictable_deinit(struct kbase_context *kctx)
 520 {
 521         unregister_shrinker(&kctx->reclaim);
 522 }
 523
 524 struct kbase_mem_zone_cache_entry {
 525         /* List head used to link the cache entry to the memory allocation. */
 526         struct list_head zone_node;
 527         /* The zone the cacheline is for. */
 528         struct zone *zone;
 529         /* The number of pages in the allocation which belong to this zone. */
 530         u64 count;
 531 };
 532
 533 static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc,
 534                 size_t start_offset)
 535 {
 536         struct kbase_mem_zone_cache_entry *cache = NULL;
 537         size_t i;
 538         int ret = 0;
 539
 540         for (i = start_offset; i < alloc->nents; i++) {
 541                 struct page *p = phys_to_page(alloc->pages[i]);
 542                 struct zone *zone = page_zone(p);
 543                 bool create = true;
 544
 545                 if (cache && (cache->zone == zone)) {
 546                         /*
 547                          * Fast path check as most of the time adjacent
 548                          * pages come from the same zone.
 549                          */
 550                         create = false;
 551                 } else {
 552                         /*
 553                          * Slow path check, walk all the cache entries to see
 554                          * if we already know about this zone.
 555                          */
 556                         list_for_each_entry(cache, &alloc->zone_cache, zone_node) {
 557                                 if (cache->zone == zone) {
 558                                         create = false;
 559                                         break;
 560                                 }
 561                         }
 562                 }
 563
 564                 /* This zone wasn't found in the cache, create an entry for it */
 565                 if (create) {
 566                         cache = kmalloc(sizeof(*cache), GFP_KERNEL);
 567                         if (!cache) {
 568                                 ret = -ENOMEM;
 569                                 goto bail;
 570                         }
 571                         cache->zone = zone;
 572                         cache->count = 0;
 573                         list_add(&cache->zone_node, &alloc->zone_cache);
 574                 }
 575
 576                 cache->count++;
 577         }
 578         return 0;
 579
 580 bail:
 581         return ret;
 582 }
 583
 584 int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
 585                 size_t start_offset)
 586 {
 587         /*
 588          * Bail if the zone cache is empty, only update the cache if it
 589          * existed in the first place.
 590          */
 591         if (list_empty(&alloc->zone_cache))
 592                 return 0;
 593
 594         return kbase_zone_cache_builder(alloc, start_offset);
 595 }
 596
 597 int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc)
 598 {
 599         /* Bail if the zone cache already exists */
 600         if (!list_empty(&alloc->zone_cache))
 601                 return 0;
 602
 603         return kbase_zone_cache_builder(alloc, 0);
 604 }
 605
 606 void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc)
 607 {
 608         struct kbase_mem_zone_cache_entry *walker;
 609
 610         while(!list_empty(&alloc->zone_cache)){
 611                 walker = list_first_entry(&alloc->zone_cache,
 612                                 struct kbase_mem_zone_cache_entry, zone_node);
 613                 list_del(&walker->zone_node);
 614                 kfree(walker);
 615         }
 616 }
 617
 618 /**
 619  * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
 620  * @alloc: The physical allocation
 621  */
 622 static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc)
 623 {
 624         struct kbase_context *kctx = alloc->imported.kctx;
 625         struct kbase_mem_zone_cache_entry *zone_cache;
 626         int __maybe_unused new_page_count;
 627         int err;
 628
 629         /* Attempt to build a zone cache of tracking */
 630         err = kbase_zone_cache_build(alloc);
 631         if (err == 0) {
 632                 /* Bulk update all the zones */
 633                 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
 634                         zone_page_state_add(zone_cache->count,
 635                                         zone_cache->zone, NR_SLAB_RECLAIMABLE);
 636                 }
 637         } else {
 638                 /* Fall-back to page by page updates */
 639                 int i;
 640
 641                 for (i = 0; i < alloc->nents; i++) {
 642                         struct page *p = phys_to_page(alloc->pages[i]);
 643                         struct zone *zone = page_zone(p);
 644
 645                         zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE);
 646                 }
 647         }
 648
 649         kbase_process_page_usage_dec(kctx, alloc->nents);
 650         new_page_count = kbase_atomic_sub_pages(alloc->nents,
 651                                                 &kctx->used_pages);
 652         kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
 653
 654         kbase_tlstream_aux_pagesalloc(
 655                         (u32)kctx->id,
 656                         (u64)new_page_count);
 657 }
 658
 659 /**
 660  * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable.
 661  * @alloc: The physical allocation
 662  */
 663 static
 664 void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc)
 665 {
 666         struct kbase_context *kctx = alloc->imported.kctx;
 667         struct kbase_mem_zone_cache_entry *zone_cache;
 668         int __maybe_unused new_page_count;
 669         int err;
 670
 671         new_page_count = kbase_atomic_add_pages(alloc->nents,
 672                                                 &kctx->used_pages);
 673         kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
 674
 675         /* Increase mm counters so that the allocation is accounted for
 676          * against the process and thus is visible to the OOM killer,
 677          * then remove it from the reclaimable accounting. */
 678         kbase_process_page_usage_inc(kctx, alloc->nents);
 679
 680         /* Attempt to build a zone cache of tracking */
 681         err = kbase_zone_cache_build(alloc);
 682         if (err == 0) {
 683                 /* Bulk update all the zones */
 684                 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
 685                         zone_page_state_add(-zone_cache->count,
 686                                         zone_cache->zone, NR_SLAB_RECLAIMABLE);
 687                 }
 688         } else {
 689                 /* Fall-back to page by page updates */
 690                 int i;
 691
 692                 for (i = 0; i < alloc->nents; i++) {
 693                         struct page *p = phys_to_page(alloc->pages[i]);
 694                         struct zone *zone = page_zone(p);
 695
 696                         zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE);
 697                 }
 698         }
 699
 700         kbase_tlstream_aux_pagesalloc(
 701                         (u32)kctx->id,
 702                         (u64)new_page_count);
 703 }
 704
 705 int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc)
 706 {
 707         struct kbase_context *kctx = gpu_alloc->imported.kctx;
 708         int err;
 709
 710         lockdep_assert_held(&kctx->reg_lock);
 711
 712         /* This alloction can't already be on a list. */
 713         WARN_ON(!list_empty(&gpu_alloc->evict_node));
 714
 715         /*
 716          * Try to shrink the CPU mappings as required, if we fail then
 717          * fail the process of making this allocation evictable.
 718          */
 719         err = kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg,
 720                         0, gpu_alloc->nents);
 721         if (err)
 722                 return -EINVAL;
 723
 724         /*
 725          * Add the allocation to the eviction list, after this point the shrink
 726          * can reclaim it.
 727          */
 728         mutex_lock(&kctx->evict_lock);
 729         list_add(&gpu_alloc->evict_node, &kctx->evict_list);
 730         mutex_unlock(&kctx->evict_lock);
 731         kbase_mem_evictable_mark_reclaim(gpu_alloc);
 732
 733         gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED;
 734         return 0;
 735 }
 736
 737 bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc)
 738 {
 739         struct kbase_context *kctx = gpu_alloc->imported.kctx;
 740         int err = 0;
 741
 742         lockdep_assert_held(&kctx->reg_lock);
 743
 744         /*
 745          * First remove the allocation from the eviction list as it's no
 746          * longer eligible for eviction.
 747          */
 748         mutex_lock(&kctx->evict_lock);
 749         list_del_init(&gpu_alloc->evict_node);
 750         mutex_unlock(&kctx->evict_lock);
 751
 752         if (gpu_alloc->evicted == 0) {
 753                 /*
 754                  * The backing is still present, update the VM stats as it's
 755                  * in use again.
 756                  */
 757                 kbase_mem_evictable_unmark_reclaim(gpu_alloc);
 758         } else {
 759                 /* If the region is still alive ... */
 760                 if (gpu_alloc->reg) {
 761                         /* ... allocate replacement backing ... */
 762                         err = kbase_alloc_phy_pages_helper(gpu_alloc,
 763                                         gpu_alloc->evicted);
 764
 765                         /*
 766                          * ... and grow the mapping back to its
 767                          * pre-eviction size.
 768                          */
 769                         if (!err)
 770                                 err = kbase_mem_grow_gpu_mapping(kctx,
 771                                                 gpu_alloc->reg,
 772                                                 gpu_alloc->evicted, 0);
 773
 774                         gpu_alloc->evicted = 0;
 775                 }
 776         }
 777
 778         /* If the region is still alive remove the DONT_NEED attribute. */
 779         if (gpu_alloc->reg)
 780                 gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED;
 781
 782         return (err == 0);
 783 }
 784
 785 int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask)
 786 {
 787         struct kbase_va_region *reg;
 788         int ret = -EINVAL;
 789         unsigned int real_flags = 0;
 790         unsigned int prev_flags = 0;
 791         bool prev_needed, new_needed;
 792
 793         KBASE_DEBUG_ASSERT(kctx);
 794
 795         if (!gpu_addr)
 796                 return -EINVAL;
 797
 798         /* nuke other bits */
 799         flags &= mask;
 800
 801         /* check for only supported flags */
 802         if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE))
 803                 goto out;
 804
 805         /* mask covers bits we don't support? */
 806         if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE))
 807                 goto out;
 808
 809         /* convert flags */
 810         if (BASE_MEM_COHERENT_SYSTEM & flags)
 811                 real_flags |= KBASE_REG_SHARE_BOTH;
 812         else if (BASE_MEM_COHERENT_LOCAL & flags)
 813                 real_flags |= KBASE_REG_SHARE_IN;
 814
 815         /* now we can lock down the context, and find the region */
 816         down_write(&current->mm->mmap_sem);
 817         kbase_gpu_vm_lock(kctx);
 818
 819         /* Validate the region */
 820         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
 821         if (!reg || (reg->flags & KBASE_REG_FREE))
 822                 goto out_unlock;
 823
 824         /* Is the region being transitioning between not needed and needed? */
 825         prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED;
 826         new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED;
 827         if (prev_needed != new_needed) {
 828                 /* Aliased allocations can't be made ephemeral */
 829                 if (atomic_read(&reg->cpu_alloc->gpu_mappings) > 1)
 830                         goto out_unlock;
 831
 832                 if (new_needed) {
 833                         /* Only native allocations can be marked not needed */
 834                         if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
 835                                 ret = -EINVAL;
 836                                 goto out_unlock;
 837                         }
 838                         ret = kbase_mem_evictable_make(reg->gpu_alloc);
 839                         if (ret)
 840                                 goto out_unlock;
 841                 } else {
 842                         kbase_mem_evictable_unmake(reg->gpu_alloc);
 843                 }
 844         }
 845
 846         /* limit to imported memory */
 847         if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
 848              (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
 849                 goto out_unlock;
 850
 851         /* no change? */
 852         if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) {
 853                 ret = 0;
 854                 goto out_unlock;
 855         }
 856
 857         /* save for roll back */
 858         prev_flags = reg->flags;
 859         reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH);
 860         reg->flags |= real_flags;
 861
 862         /* Currently supporting only imported memory */
 863         switch (reg->gpu_alloc->type) {
 864 #ifdef CONFIG_UMP
 865         case KBASE_MEM_TYPE_IMPORTED_UMP:
 866                 ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags);
 867                 break;
 868 #endif
 869 #ifdef CONFIG_DMA_SHARED_BUFFER
 870         case KBASE_MEM_TYPE_IMPORTED_UMM:
 871                 /* Future use will use the new flags, existing mapping will NOT be updated
 872                  * as memory should not be in use by the GPU when updating the flags.
 873                  */
 874                 ret = 0;
 875                 WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count);
 876                 break;
 877 #endif
 878         default:
 879                 break;
 880         }
 881
 882         /* roll back on error, i.e. not UMP */
 883         if (ret)
 884                 reg->flags = prev_flags;
 885
 886 out_unlock:
 887         kbase_gpu_vm_unlock(kctx);
 888         up_write(&current->mm->mmap_sem);
 889 out:
 890         return ret;
 891 }
 892
 893 #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS)
 894
 895 #ifdef CONFIG_UMP
 896 static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags)
 897 {
 898         struct kbase_va_region *reg;
 899         ump_dd_handle umph;
 900         u64 block_count;
 901         const ump_dd_physical_block_64 *block_array;
 902         u64 i, j;
 903         int page = 0;
 904         ump_alloc_flags ump_flags;
 905         ump_alloc_flags cpu_flags;
 906         ump_alloc_flags gpu_flags;
 907
 908         if (*flags & BASE_MEM_SECURE)
 909                 goto bad_flags;
 910
 911         umph = ump_dd_from_secure_id(id);
 912         if (UMP_DD_INVALID_MEMORY_HANDLE == umph)
 913                 goto bad_id;
 914
 915         ump_flags = ump_dd_allocation_flags_get(umph);
 916         cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK;
 917         gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) &
 918                         UMP_DEVICE_MASK;
 919
 920         *va_pages = ump_dd_size_get_64(umph);
 921         *va_pages >>= PAGE_SHIFT;
 922
 923         if (!*va_pages)
 924                 goto bad_size;
 925
 926         if (*va_pages > (U64_MAX / PAGE_SIZE))
 927                 /* 64-bit address range is the max */
 928                 goto bad_size;
 929
 930         if (*flags & BASE_MEM_SAME_VA)
 931                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
 932         else
 933                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
 934
 935         if (!reg)
 936                 goto no_region;
 937
 938         /* we've got pages to map now, and support SAME_VA */
 939         *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
 940
 941         reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
 942         if (IS_ERR_OR_NULL(reg->gpu_alloc))
 943                 goto no_alloc_obj;
 944
 945         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
 946
 947         reg->gpu_alloc->imported.ump_handle = umph;
 948
 949         reg->flags &= ~KBASE_REG_FREE;
 950         reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */
 951         reg->flags &= ~KBASE_REG_GROWABLE;      /* UMP cannot be grown */
 952
 953         /* Override import flags based on UMP flags */
 954         *flags &= ~(BASE_MEM_CACHED_CPU);
 955         *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR);
 956         *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR);
 957
 958         if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
 959             (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) {
 960                 reg->flags |= KBASE_REG_CPU_CACHED;
 961                 *flags |= BASE_MEM_CACHED_CPU;
 962         }
 963
 964         if (cpu_flags & UMP_PROT_CPU_WR) {
 965                 reg->flags |= KBASE_REG_CPU_WR;
 966                 *flags |= BASE_MEM_PROT_CPU_WR;
 967         }
 968
 969         if (cpu_flags & UMP_PROT_CPU_RD) {
 970                 reg->flags |= KBASE_REG_CPU_RD;
 971                 *flags |= BASE_MEM_PROT_CPU_RD;
 972         }
 973
 974         if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
 975             (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR))
 976                 reg->flags |= KBASE_REG_GPU_CACHED;
 977
 978         if (gpu_flags & UMP_PROT_DEVICE_WR) {
 979                 reg->flags |= KBASE_REG_GPU_WR;
 980                 *flags |= BASE_MEM_PROT_GPU_WR;
 981         }
 982
 983         if (gpu_flags & UMP_PROT_DEVICE_RD) {
 984                 reg->flags |= KBASE_REG_GPU_RD;
 985                 *flags |= BASE_MEM_PROT_GPU_RD;
 986         }
 987
 988         /* ump phys block query */
 989         ump_dd_phys_blocks_get_64(umph, &block_count, &block_array);
 990
 991         for (i = 0; i < block_count; i++) {
 992                 for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) {
 993                         reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
 994                         page++;
 995                 }
 996         }
 997         reg->gpu_alloc->nents = *va_pages;
 998         reg->extent = 0;
 999
1000         return reg;
1001
1002 no_alloc_obj:
1003         kfree(reg);
1004 no_region:
1005 bad_size:
1006         ump_dd_release(umph);
1007 bad_id:
1008 bad_flags:
1009         return NULL;
1010 }
1011 #endif                          /* CONFIG_UMP */
1012
1013 #ifdef CONFIG_DMA_SHARED_BUFFER
1014 static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags)
1015 {
1016         struct kbase_va_region *reg;
1017         struct dma_buf *dma_buf;
1018         struct dma_buf_attachment *dma_attachment;
1019         bool shared_zone = false;
1020
1021         dma_buf = dma_buf_get(fd);
1022         if (IS_ERR_OR_NULL(dma_buf))
1023                 goto no_buf;
1024
1025         dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev);
1026         if (!dma_attachment)
1027                 goto no_attachment;
1028
1029         *va_pages = PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT;
1030         if (!*va_pages)
1031                 goto bad_size;
1032
1033         if (*va_pages > (U64_MAX / PAGE_SIZE))
1034                 /* 64-bit address range is the max */
1035                 goto bad_size;
1036
1037         /* ignore SAME_VA */
1038         *flags &= ~BASE_MEM_SAME_VA;
1039
1040         if (*flags & BASE_MEM_IMPORT_SHARED)
1041                 shared_zone = true;
1042
1043 #ifdef CONFIG_64BIT
1044         if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1045                 /*
1046                  * 64-bit tasks require us to reserve VA on the CPU that we use
1047                  * on the GPU.
1048                  */
1049                 shared_zone = true;
1050         }
1051 #endif
1052
1053         if (shared_zone) {
1054                 *flags |= BASE_MEM_NEED_MMAP;
1055                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
1056         } else {
1057                 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
1058         }
1059
1060         if (!reg)
1061                 goto no_region;
1062
1063         reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
1064         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1065                 goto no_alloc_obj;
1066
1067         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1068
1069         /* No pages to map yet */
1070         reg->gpu_alloc->nents = 0;
1071
1072         reg->flags &= ~KBASE_REG_FREE;
1073         reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */
1074         reg->flags &= ~KBASE_REG_GROWABLE;      /* UMM cannot be grown */
1075         reg->flags |= KBASE_REG_GPU_CACHED;
1076
1077         if (*flags & BASE_MEM_PROT_CPU_WR)
1078                 reg->flags |= KBASE_REG_CPU_WR;
1079
1080         if (*flags & BASE_MEM_PROT_CPU_RD)
1081                 reg->flags |= KBASE_REG_CPU_RD;
1082
1083         if (*flags & BASE_MEM_PROT_GPU_WR)
1084                 reg->flags |= KBASE_REG_GPU_WR;
1085
1086         if (*flags & BASE_MEM_PROT_GPU_RD)
1087                 reg->flags |= KBASE_REG_GPU_RD;
1088
1089         if (*flags & BASE_MEM_SECURE)
1090                 reg->flags |= KBASE_REG_SECURE;
1091
1092         /* no read or write permission given on import, only on run do we give the right permissions */
1093
1094         reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM;
1095         reg->gpu_alloc->imported.umm.sgt = NULL;
1096         reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
1097         reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment;
1098         reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0;
1099         reg->extent = 0;
1100
1101         return reg;
1102
1103 no_alloc_obj:
1104         kfree(reg);
1105 no_region:
1106 bad_size:
1107         dma_buf_detach(dma_buf, dma_attachment);
1108 no_attachment:
1109         dma_buf_put(dma_buf);
1110 no_buf:
1111         return NULL;
1112 }
1113 #endif  /* CONFIG_DMA_SHARED_BUFFER */
1114
1115
1116 static struct kbase_va_region *kbase_mem_from_user_buffer(
1117                 struct kbase_context *kctx, unsigned long address,
1118                 unsigned long size, u64 *va_pages, u64 *flags)
1119 {
1120         struct kbase_va_region *reg;
1121         long faulted_pages;
1122         int zone = KBASE_REG_ZONE_CUSTOM_VA;
1123         bool shared_zone = false;
1124
1125         *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) -
1126                 PFN_DOWN(address);
1127         if (!*va_pages)
1128                 goto bad_size;
1129
1130         if (*va_pages > (UINT64_MAX / PAGE_SIZE))
1131                 /* 64-bit address range is the max */
1132                 goto bad_size;
1133
1134         /* SAME_VA generally not supported with imported memory (no known use cases) */
1135         *flags &= ~BASE_MEM_SAME_VA;
1136
1137         if (*flags & BASE_MEM_IMPORT_SHARED)
1138                 shared_zone = true;
1139
1140 #ifdef CONFIG_64BIT
1141         if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1142                 /*
1143                  * 64-bit tasks require us to reserve VA on the CPU that we use
1144                  * on the GPU.
1145                  */
1146                 shared_zone = true;
1147         }
1148 #endif
1149
1150         if (shared_zone) {
1151                 *flags |= BASE_MEM_NEED_MMAP;
1152                 zone = KBASE_REG_ZONE_SAME_VA;
1153         }
1154
1155         reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone);
1156
1157         if (!reg)
1158                 goto no_region;
1159
1160         reg->gpu_alloc = kbase_alloc_create(*va_pages,
1161                         KBASE_MEM_TYPE_IMPORTED_USER_BUF);
1162         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1163                 goto no_alloc_obj;
1164
1165         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1166
1167         reg->flags &= ~KBASE_REG_FREE;
1168         reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */
1169         reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */
1170
1171         if (*flags & BASE_MEM_PROT_CPU_WR)
1172                 reg->flags |= KBASE_REG_CPU_WR;
1173
1174         if (*flags & BASE_MEM_PROT_CPU_RD)
1175                 reg->flags |= KBASE_REG_CPU_RD;
1176
1177         if (*flags & BASE_MEM_PROT_GPU_WR)
1178                 reg->flags |= KBASE_REG_GPU_WR;
1179
1180         if (*flags & BASE_MEM_PROT_GPU_RD)
1181                 reg->flags |= KBASE_REG_GPU_RD;
1182
1183         down_read(&current->mm->mmap_sem);
1184
1185         /* A sanity check that get_user_pages will work on the memory */
1186         /* (so the initial import fails on weird memory regions rather than */
1187         /* the job failing when we try to handle the external resources). */
1188         /* It doesn't take a reference to the pages (because the page list is NULL). */
1189         /* We can't really store the page list because that would involve */
1190         /* keeping the pages pinned - instead we pin/unpin around the job */
1191         /* (as part of the external resources handling code) */
1192 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1193         faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
1194                         reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1195 #else
1196         faulted_pages = get_user_pages(address, *va_pages,
1197                         reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1198 #endif
1199         up_read(&current->mm->mmap_sem);
1200
1201         if (faulted_pages != *va_pages)
1202                 goto fault_mismatch;
1203
1204         reg->gpu_alloc->imported.user_buf.size = size;
1205         reg->gpu_alloc->imported.user_buf.address = address;
1206         reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages;
1207         reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages,
1208                         sizeof(struct page *), GFP_KERNEL);
1209         reg->gpu_alloc->imported.user_buf.mm = current->mm;
1210         atomic_inc(&current->mm->mm_count);
1211
1212         if (!reg->gpu_alloc->imported.user_buf.pages)
1213                 goto no_page_array;
1214
1215         reg->gpu_alloc->nents = 0;
1216         reg->extent = 0;
1217
1218         return reg;
1219
1220 no_page_array:
1221 fault_mismatch:
1222         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1223 no_alloc_obj:
1224         kfree(reg);
1225 no_region:
1226 bad_size:
1227         return NULL;
1228
1229 }
1230
1231
1232 u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride,
1233                     u64 nents, struct base_mem_aliasing_info *ai,
1234                     u64 *num_pages)
1235 {
1236         struct kbase_va_region *reg;
1237         u64 gpu_va;
1238         size_t i;
1239         bool coherent;
1240
1241         KBASE_DEBUG_ASSERT(kctx);
1242         KBASE_DEBUG_ASSERT(flags);
1243         KBASE_DEBUG_ASSERT(ai);
1244         KBASE_DEBUG_ASSERT(num_pages);
1245
1246         /* mask to only allowed flags */
1247         *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
1248                    BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL |
1249                    BASE_MEM_COHERENT_SYSTEM_REQUIRED);
1250
1251         if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
1252                 dev_warn(kctx->kbdev->dev,
1253                                 "kbase_mem_alias called with bad flags (%llx)",
1254                                 (unsigned long long)*flags);
1255                 goto bad_flags;
1256         }
1257         coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 ||
1258                         (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0;
1259
1260         if (!stride)
1261                 goto bad_stride;
1262
1263         if (!nents)
1264                 goto bad_nents;
1265
1266         if ((nents * stride) > (U64_MAX / PAGE_SIZE))
1267                 /* 64-bit address range is the max */
1268                 goto bad_size;
1269
1270         /* calculate the number of pages this alias will cover */
1271         *num_pages = nents * stride;
1272
1273 #ifdef CONFIG_64BIT
1274         if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1275                 /* 64-bit tasks must MMAP anyway, but not expose this address to
1276                  * clients */
1277                 *flags |= BASE_MEM_NEED_MMAP;
1278                 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1279                                               KBASE_REG_ZONE_SAME_VA);
1280         } else {
1281 #else
1282         if (1) {
1283 #endif
1284                 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1285                                               KBASE_REG_ZONE_CUSTOM_VA);
1286         }
1287
1288         if (!reg)
1289                 goto no_reg;
1290
1291         /* zero-sized page array, as we don't need one/can support one */
1292         reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
1293         if (IS_ERR_OR_NULL(reg->gpu_alloc))
1294                 goto no_alloc_obj;
1295
1296         reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1297
1298         kbase_update_region_flags(kctx, reg, *flags);
1299
1300         reg->gpu_alloc->imported.alias.nents = nents;
1301         reg->gpu_alloc->imported.alias.stride = stride;
1302         reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents);
1303         if (!reg->gpu_alloc->imported.alias.aliased)
1304                 goto no_aliased_array;
1305
1306         kbase_gpu_vm_lock(kctx);
1307
1308         /* validate and add src handles */
1309         for (i = 0; i < nents; i++) {
1310                 if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) {
1311                         if (ai[i].handle.basep.handle !=
1312                             BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE)
1313                                 goto bad_handle; /* unsupported magic handle */
1314                         if (!ai[i].length)
1315                                 goto bad_handle; /* must be > 0 */
1316                         if (ai[i].length > stride)
1317                                 goto bad_handle; /* can't be larger than the
1318                                                     stride */
1319                         reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1320                 } else {
1321                         struct kbase_va_region *aliasing_reg;
1322                         struct kbase_mem_phy_alloc *alloc;
1323
1324                         aliasing_reg = kbase_region_tracker_find_region_base_address(
1325                                 kctx,
1326                                 (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT);
1327
1328                         /* validate found region */
1329                         if (!aliasing_reg)
1330                                 goto bad_handle; /* Not found */
1331                         if (aliasing_reg->flags & KBASE_REG_FREE)
1332                                 goto bad_handle; /* Free region */
1333                         if (aliasing_reg->flags & KBASE_REG_DONT_NEED)
1334                                 goto bad_handle; /* Ephemeral region */
1335                         if (!aliasing_reg->gpu_alloc)
1336                                 goto bad_handle; /* No alloc */
1337                         if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
1338                                 goto bad_handle; /* Not a native alloc */
1339                         if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0))
1340                                 goto bad_handle;
1341                                 /* Non-coherent memory cannot alias
1342                                    coherent memory, and vice versa.*/
1343
1344                         /* check size against stride */
1345                         if (!ai[i].length)
1346                                 goto bad_handle; /* must be > 0 */
1347                         if (ai[i].length > stride)
1348                                 goto bad_handle; /* can't be larger than the
1349                                                     stride */
1350
1351                         alloc = aliasing_reg->gpu_alloc;
1352
1353                         /* check against the alloc's size */
1354                         if (ai[i].offset > alloc->nents)
1355                                 goto bad_handle; /* beyond end */
1356                         if (ai[i].offset + ai[i].length > alloc->nents)
1357                                 goto bad_handle; /* beyond end */
1358
1359                         reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
1360                         reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1361                         reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset;
1362                 }
1363         }
1364
1365 #ifdef CONFIG_64BIT
1366         if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1367                 /* Bind to a cookie */
1368                 if (!kctx->cookies) {
1369                         dev_err(kctx->kbdev->dev, "No cookies available for allocation!");
1370                         goto no_cookie;
1371                 }
1372                 /* return a cookie */
1373                 gpu_va = __ffs(kctx->cookies);
1374                 kctx->cookies &= ~(1UL << gpu_va);
1375                 BUG_ON(kctx->pending_regions[gpu_va]);
1376                 kctx->pending_regions[gpu_va] = reg;
1377
1378                 /* relocate to correct base */
1379                 gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1380                 gpu_va <<= PAGE_SHIFT;
1381         } else /* we control the VA */ {
1382 #else
1383         if (1) {
1384 #endif
1385                 if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
1386                         dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
1387                         goto no_mmap;
1388                 }
1389                 /* return real GPU VA */
1390                 gpu_va = reg->start_pfn << PAGE_SHIFT;
1391         }
1392
1393         reg->flags &= ~KBASE_REG_FREE;
1394         reg->flags &= ~KBASE_REG_GROWABLE;
1395
1396         kbase_gpu_vm_unlock(kctx);
1397
1398         return gpu_va;
1399
1400 #ifdef CONFIG_64BIT
1401 no_cookie:
1402 #endif
1403 no_mmap:
1404 bad_handle:
1405         kbase_gpu_vm_unlock(kctx);
1406 no_aliased_array:
1407         kbase_mem_phy_alloc_put(reg->cpu_alloc);
1408         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1409 no_alloc_obj:
1410         kfree(reg);
1411 no_reg:
1412 bad_size:
1413 bad_nents:
1414 bad_stride:
1415 bad_flags:
1416         return 0;
1417 }
1418
1419 static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx)
1420 {
1421         u32 cpu_cache_line_size = cache_line_size();
1422         u32 gpu_cache_line_size =
1423                 (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size);
1424
1425         return ((cpu_cache_line_size > gpu_cache_line_size) ?
1426                                 cpu_cache_line_size :
1427                                 gpu_cache_line_size);
1428 }
1429
1430 static int kbase_check_buffer_size(struct kbase_context *kctx, u64 size)
1431 {
1432         u32 cache_line_align = kbase_get_cache_line_alignment(kctx);
1433
1434         return (size & (cache_line_align - 1)) == 0 ? 0 : -EINVAL;
1435 }
1436
1437 static int kbase_check_buffer_cache_alignment(struct kbase_context *kctx,
1438                                         void __user *ptr)
1439 {
1440         u32 cache_line_align = kbase_get_cache_line_alignment(kctx);
1441
1442         return ((uintptr_t)ptr & (cache_line_align - 1)) == 0 ? 0 : -EINVAL;
1443 }
1444
1445 int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
1446                 void __user *phandle, u64 *gpu_va, u64 *va_pages,
1447                 u64 *flags)
1448 {
1449         struct kbase_va_region *reg;
1450
1451         KBASE_DEBUG_ASSERT(kctx);
1452         KBASE_DEBUG_ASSERT(gpu_va);
1453         KBASE_DEBUG_ASSERT(va_pages);
1454         KBASE_DEBUG_ASSERT(flags);
1455
1456 #ifdef CONFIG_64BIT
1457         if (!kbase_ctx_flag(kctx, KCTX_COMPAT))
1458                 *flags |= BASE_MEM_SAME_VA;
1459 #endif
1460
1461         if (!kbase_check_import_flags(*flags)) {
1462                 dev_warn(kctx->kbdev->dev,
1463                                 "kbase_mem_import called with bad flags (%llx)",
1464                                 (unsigned long long)*flags);
1465                 goto bad_flags;
1466         }
1467
1468         switch (type) {
1469 #ifdef CONFIG_UMP
1470         case BASE_MEM_IMPORT_TYPE_UMP: {
1471                 ump_secure_id id;
1472
1473                 if (get_user(id, (ump_secure_id __user *)phandle))
1474                         reg = NULL;
1475                 else
1476                         reg = kbase_mem_from_ump(kctx, id, va_pages, flags);
1477         }
1478         break;
1479 #endif /* CONFIG_UMP */
1480 #ifdef CONFIG_DMA_SHARED_BUFFER
1481         case BASE_MEM_IMPORT_TYPE_UMM: {
1482                 int fd;
1483
1484                 if (get_user(fd, (int __user *)phandle))
1485                         reg = NULL;
1486                 else
1487                         reg = kbase_mem_from_umm(kctx, fd, va_pages, flags);
1488         }
1489         break;
1490 #endif /* CONFIG_DMA_SHARED_BUFFER */
1491         case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
1492                 struct base_mem_import_user_buffer user_buffer;
1493                 void __user *uptr;
1494
1495                 if (copy_from_user(&user_buffer, phandle,
1496                                 sizeof(user_buffer))) {
1497                         reg = NULL;
1498                 } else {
1499 #ifdef CONFIG_COMPAT
1500                         if (kbase_ctx_flag(kctx, KCTX_COMPAT))
1501                                 uptr = compat_ptr(user_buffer.ptr.compat_value);
1502                         else
1503 #endif
1504                                 uptr = user_buffer.ptr.value;
1505
1506                         if (0 != kbase_check_buffer_cache_alignment(kctx,
1507                                                                         uptr)) {
1508                                 dev_warn(kctx->kbdev->dev,
1509                                 "User buffer is not cache line aligned!\n");
1510                                 goto no_reg;
1511                         }
1512
1513                         if (0 != kbase_check_buffer_size(kctx,
1514                                         user_buffer.length)) {
1515                                 dev_warn(kctx->kbdev->dev,
1516                                 "User buffer size is not multiple of cache line size!\n");
1517                                 goto no_reg;
1518                         }
1519
1520                         reg = kbase_mem_from_user_buffer(kctx,
1521                                         (unsigned long)uptr, user_buffer.length,
1522                                         va_pages, flags);
1523                 }
1524                 break;
1525         }
1526         default: {
1527                 reg = NULL;
1528                 break;
1529         }
1530         }
1531
1532         if (!reg)
1533                 goto no_reg;
1534
1535         kbase_gpu_vm_lock(kctx);
1536
1537         /* mmap needed to setup VA? */
1538         if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) {
1539                 /* Bind to a cookie */
1540                 if (!kctx->cookies)
1541                         goto no_cookie;
1542                 /* return a cookie */
1543                 *gpu_va = __ffs(kctx->cookies);
1544                 kctx->cookies &= ~(1UL << *gpu_va);
1545                 BUG_ON(kctx->pending_regions[*gpu_va]);
1546                 kctx->pending_regions[*gpu_va] = reg;
1547
1548                 /* relocate to correct base */
1549                 *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1550                 *gpu_va <<= PAGE_SHIFT;
1551
1552         } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES)  {
1553                 /* we control the VA, mmap now to the GPU */
1554                 if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
1555                         goto no_gpu_va;
1556                 /* return real GPU VA */
1557                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1558         } else {
1559                 /* we control the VA, but nothing to mmap yet */
1560                 if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0)
1561                         goto no_gpu_va;
1562                 /* return real GPU VA */
1563                 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1564         }
1565
1566         /* clear out private flags */
1567         *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1);
1568
1569         kbase_gpu_vm_unlock(kctx);
1570
1571         return 0;
1572
1573 no_gpu_va:
1574 no_cookie:
1575         kbase_gpu_vm_unlock(kctx);
1576         kbase_mem_phy_alloc_put(reg->cpu_alloc);
1577         kbase_mem_phy_alloc_put(reg->gpu_alloc);
1578         kfree(reg);
1579 no_reg:
1580 bad_flags:
1581         *gpu_va = 0;
1582         *va_pages = 0;
1583         *flags = 0;
1584         return -ENOMEM;
1585 }
1586
1587
1588 static int zap_range_nolock(struct mm_struct *mm,
1589                 const struct vm_operations_struct *vm_ops,
1590                 unsigned long start, unsigned long end)
1591 {
1592         struct vm_area_struct *vma;
1593         int err = -EINVAL; /* in case end < start */
1594
1595         while (start < end) {
1596                 unsigned long local_start;
1597                 unsigned long local_end;
1598
1599                 vma = find_vma_intersection(mm, start, end);
1600                 if (!vma)
1601                         break;
1602
1603                 /* is it ours? */
1604                 if (vma->vm_ops != vm_ops)
1605                         goto try_next;
1606
1607                 local_start = vma->vm_start;
1608
1609                 if (start > local_start)
1610                         local_start = start;
1611
1612                 local_end = vma->vm_end;
1613
1614                 if (end < local_end)
1615                         local_end = end;
1616
1617                 err = zap_vma_ptes(vma, local_start, local_end - local_start);
1618                 if (unlikely(err))
1619                         break;
1620
1621 try_next:
1622                 /* go to next vma, if any */
1623                 start = vma->vm_end;
1624         }
1625
1626         return err;
1627 }
1628
1629 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
1630                 struct kbase_va_region *reg,
1631                 u64 new_pages, u64 old_pages)
1632 {
1633         phys_addr_t *phy_pages;
1634         u64 delta = new_pages - old_pages;
1635         int ret = 0;
1636
1637         lockdep_assert_held(&kctx->reg_lock);
1638
1639         /* Map the new pages into the GPU */
1640         phy_pages = kbase_get_gpu_phy_pages(reg);
1641         ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages,
1642                         phy_pages + old_pages, delta, reg->flags);
1643
1644         return ret;
1645 }
1646
1647 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
1648                 struct kbase_va_region *reg,
1649                 u64 new_pages, u64 old_pages)
1650 {
1651         struct kbase_mem_phy_alloc *cpu_alloc = reg->cpu_alloc;
1652         struct kbase_cpu_mapping *mapping;
1653         int err;
1654
1655         lockdep_assert_held(&kctx->process_mm->mmap_sem);
1656
1657         list_for_each_entry(mapping, &cpu_alloc->mappings, mappings_list) {
1658                 unsigned long mapping_size;
1659
1660                 mapping_size = (mapping->vm_end - mapping->vm_start)
1661                                 >> PAGE_SHIFT;
1662
1663                 /* is this mapping affected ?*/
1664                 if ((mapping->page_off + mapping_size) > new_pages) {
1665                         unsigned long first_bad = 0;
1666
1667                         if (new_pages > mapping->page_off)
1668                                 first_bad = new_pages - mapping->page_off;
1669
1670                         err = zap_range_nolock(current->mm,
1671                                         &kbase_vm_ops,
1672                                         mapping->vm_start +
1673                                         (first_bad << PAGE_SHIFT),
1674                                         mapping->vm_end);
1675
1676                         WARN(err,
1677                              "Failed to zap VA range (0x%lx - 0x%lx);\n",
1678                              mapping->vm_start +
1679                              (first_bad << PAGE_SHIFT),
1680                              mapping->vm_end
1681                              );
1682
1683                         /* The zap failed, give up and exit */
1684                         if (err)
1685                                 goto failed;
1686                 }
1687         }
1688
1689         return 0;
1690
1691 failed:
1692         return err;
1693 }
1694
1695 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
1696                 struct kbase_va_region *reg,
1697                 u64 new_pages, u64 old_pages)
1698 {
1699         u64 delta = old_pages - new_pages;
1700         int ret = 0;
1701
1702         ret = kbase_mmu_teardown_pages(kctx,
1703                         reg->start_pfn + new_pages, delta);
1704
1705         return ret;
1706 }
1707
1708 int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
1709 {
1710         u64 old_pages;
1711         u64 delta;
1712         int res = -EINVAL;
1713         struct kbase_va_region *reg;
1714         bool read_locked = false;
1715
1716         KBASE_DEBUG_ASSERT(kctx);
1717         KBASE_DEBUG_ASSERT(failure_reason);
1718         KBASE_DEBUG_ASSERT(gpu_addr != 0);
1719
1720         down_write(&current->mm->mmap_sem);
1721         kbase_gpu_vm_lock(kctx);
1722
1723         /* Validate the region */
1724         reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1725         if (!reg || (reg->flags & KBASE_REG_FREE)) {
1726                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1727                 goto out_unlock;
1728         }
1729
1730         KBASE_DEBUG_ASSERT(reg->cpu_alloc);
1731         KBASE_DEBUG_ASSERT(reg->gpu_alloc);
1732
1733         if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
1734                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1735                 goto out_unlock;
1736         }
1737
1738         if (0 == (reg->flags & KBASE_REG_GROWABLE)) {
1739                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1740                 goto out_unlock;
1741         }
1742
1743         if (new_pages > reg->nr_pages) {
1744                 /* Would overflow the VA region */
1745                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1746                 goto out_unlock;
1747         }
1748
1749         /* can't be mapped more than once on the GPU */
1750         if (atomic_read(&reg->gpu_alloc->gpu_mappings) > 1) {
1751                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1752                 goto out_unlock;
1753         }
1754         /* can't grow regions which are ephemeral */
1755         if (reg->flags & KBASE_REG_DONT_NEED) {
1756                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1757                 goto out_unlock;
1758         }
1759
1760         if (new_pages == reg->gpu_alloc->nents) {
1761                 /* no change */
1762                 res = 0;
1763                 goto out_unlock;
1764         }
1765
1766         old_pages = kbase_reg_current_backed_size(reg);
1767         if (new_pages > old_pages) {
1768                 delta = new_pages - old_pages;
1769
1770                 /*
1771                  * No update to the mm so downgrade the writer lock to a read
1772                  * lock so other readers aren't blocked after this point.
1773                  */
1774                 downgrade_write(&current->mm->mmap_sem);
1775                 read_locked = true;
1776
1777                 /* Allocate some more pages */
1778                 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
1779                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1780                         goto out_unlock;
1781                 }
1782                 if (reg->cpu_alloc != reg->gpu_alloc) {
1783                         if (kbase_alloc_phy_pages_helper(
1784                                         reg->gpu_alloc, delta) != 0) {
1785                                 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1786                                 kbase_free_phy_pages_helper(reg->cpu_alloc,
1787                                                 delta);
1788                                 goto out_unlock;
1789                         }
1790                 }
1791
1792                 /* No update required for CPU mappings, that's done on fault. */
1793
1794                 /* Update GPU mapping. */
1795                 res = kbase_mem_grow_gpu_mapping(kctx, reg,
1796                                 new_pages, old_pages);
1797
1798                 /* On error free the new pages */
1799                 if (res) {
1800                         kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1801                         if (reg->cpu_alloc != reg->gpu_alloc)
1802                                 kbase_free_phy_pages_helper(reg->gpu_alloc,
1803                                                 delta);
1804                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1805                         goto out_unlock;
1806                 }
1807         } else {
1808                 delta = old_pages - new_pages;
1809
1810                 /* Update all CPU mapping(s) */
1811                 res = kbase_mem_shrink_cpu_mapping(kctx, reg,
1812                                 new_pages, old_pages);
1813                 if (res) {
1814                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1815                         goto out_unlock;
1816                 }
1817
1818                 /* Update the GPU mapping */
1819                 res = kbase_mem_shrink_gpu_mapping(kctx, reg,
1820                                 new_pages, old_pages);
1821                 if (res) {
1822                         *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1823                         goto out_unlock;
1824                 }
1825
1826                 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1827                 if (reg->cpu_alloc != reg->gpu_alloc)
1828                         kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
1829         }
1830
1831 out_unlock:
1832         kbase_gpu_vm_unlock(kctx);
1833         if (read_locked)
1834                 up_read(&current->mm->mmap_sem);
1835         else
1836                 up_write(&current->mm->mmap_sem);
1837
1838         return res;
1839 }
1840
1841 static void kbase_cpu_vm_open(struct vm_area_struct *vma)
1842 {
1843         struct kbase_cpu_mapping *map = vma->vm_private_data;
1844
1845         KBASE_DEBUG_ASSERT(map);
1846         KBASE_DEBUG_ASSERT(map->count > 0);
1847         /* non-atomic as we're under Linux' mm lock */
1848         map->count++;
1849 }
1850
1851 static void kbase_cpu_vm_close(struct vm_area_struct *vma)
1852 {
1853         struct kbase_cpu_mapping *map = vma->vm_private_data;
1854
1855         KBASE_DEBUG_ASSERT(map);
1856         KBASE_DEBUG_ASSERT(map->count > 0);
1857
1858         /* non-atomic as we're under Linux' mm lock */
1859         if (--map->count)
1860                 return;
1861
1862         KBASE_DEBUG_ASSERT(map->kctx);
1863         KBASE_DEBUG_ASSERT(map->alloc);
1864
1865         kbase_gpu_vm_lock(map->kctx);
1866
1867         if (map->region) {
1868                 KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) ==
1869                                 KBASE_REG_ZONE_SAME_VA);
1870                 /* Avoid freeing memory on the process death which results in
1871                  * GPU Page Fault. Memory will be freed in kbase_destroy_context
1872                  */
1873                 if (!(current->flags & PF_EXITING))
1874                         kbase_mem_free_region(map->kctx, map->region);
1875         }
1876
1877         list_del(&map->mappings_list);
1878
1879         kbase_gpu_vm_unlock(map->kctx);
1880
1881         kbase_mem_phy_alloc_put(map->alloc);
1882         kfree(map);
1883 }
1884
1885 KBASE_EXPORT_TEST_API(kbase_cpu_vm_close);
1886
1887
1888 static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1889 {
1890         struct kbase_cpu_mapping *map = vma->vm_private_data;
1891         pgoff_t rel_pgoff;
1892         size_t i;
1893
1894         KBASE_DEBUG_ASSERT(map);
1895         KBASE_DEBUG_ASSERT(map->count > 0);
1896         KBASE_DEBUG_ASSERT(map->kctx);
1897         KBASE_DEBUG_ASSERT(map->alloc);
1898
1899         /* we don't use vmf->pgoff as it's affected by our mmap with
1900          * offset being a GPU VA or a cookie */
1901         rel_pgoff = ((unsigned long)vmf->virtual_address - map->vm_start)
1902                         >> PAGE_SHIFT;
1903
1904         kbase_gpu_vm_lock(map->kctx);
1905         if (map->page_off + rel_pgoff >= map->alloc->nents)
1906                 goto locked_bad_fault;
1907
1908         /* Fault on access to DONT_NEED regions */
1909         if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED))
1910                 goto locked_bad_fault;
1911
1912         /* insert all valid pages from the fault location */
1913         for (i = rel_pgoff;
1914              i < MIN((vma->vm_end - vma->vm_start) >> PAGE_SHIFT,
1915              map->alloc->nents - map->page_off); i++) {
1916                 int ret = vm_insert_pfn(vma, map->vm_start + (i << PAGE_SHIFT),
1917                     PFN_DOWN(map->alloc->pages[map->page_off + i]));
1918                 if (ret < 0 && ret != -EBUSY)
1919                         goto locked_bad_fault;
1920         }
1921
1922         kbase_gpu_vm_unlock(map->kctx);
1923         /* we resolved it, nothing for VM to do */
1924         return VM_FAULT_NOPAGE;
1925
1926 locked_bad_fault:
1927         kbase_gpu_vm_unlock(map->kctx);
1928         return VM_FAULT_SIGBUS;
1929 }
1930
1931 static const struct vm_operations_struct kbase_vm_ops = {
1932         .open  = kbase_cpu_vm_open,
1933         .close = kbase_cpu_vm_close,
1934         .fault = kbase_cpu_vm_fault
1935 };
1936
1937 static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close)
1938 {
1939         struct kbase_cpu_mapping *map;
1940         u64 start_off = vma->vm_pgoff - reg->start_pfn;
1941         phys_addr_t *page_array;
1942         int err = 0;
1943         int i;
1944
1945         map = kzalloc(sizeof(*map), GFP_KERNEL);
1946
1947         if (!map) {
1948                 WARN_ON(1);
1949                 err = -ENOMEM;
1950                 goto out;
1951         }
1952
1953         /*
1954          * VM_DONTCOPY - don't make this mapping available in fork'ed processes
1955          * VM_DONTEXPAND - disable mremap on this region
1956          * VM_IO - disables paging
1957          * VM_DONTDUMP - Don't include in core dumps (3.7 only)
1958          * VM_MIXEDMAP - Support mixing struct page*s and raw pfns.
1959          *               This is needed to support using the dedicated and
1960          *               the OS based memory backends together.
1961          */
1962         /*
1963          * This will need updating to propagate coherency flags
1964          * See MIDBASE-1057
1965          */
1966
1967 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
1968         vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO;
1969 #else
1970         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
1971 #endif
1972         vma->vm_ops = &kbase_vm_ops;
1973         vma->vm_private_data = map;
1974
1975         page_array = kbase_get_cpu_phy_pages(reg);
1976
1977         if (!(reg->flags & KBASE_REG_CPU_CACHED) &&
1978             (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) {
1979                 /* We can't map vmalloc'd memory uncached.
1980                  * Other memory will have been returned from
1981                  * kbase_mem_pool which would be
1982                  * suitable for mapping uncached.
1983                  */
1984                 BUG_ON(kaddr);
1985                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1986         }
1987
1988         if (!kaddr) {
1989                 unsigned long addr = vma->vm_start + aligned_offset;
1990
1991                 vma->vm_flags |= VM_PFNMAP;
1992                 for (i = 0; i < nr_pages; i++) {
1993                         unsigned long pfn = PFN_DOWN(page_array[i + start_off]);
1994
1995                         err = vm_insert_pfn(vma, addr, pfn);
1996                         if (WARN_ON(err))
1997                                 break;
1998
1999                         addr += PAGE_SIZE;
2000                 }
2001         } else {
2002                 WARN_ON(aligned_offset);
2003                 /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */
2004                 vma->vm_flags |= VM_MIXEDMAP;
2005                 /* vmalloc remaping is easy... */
2006                 err = remap_vmalloc_range(vma, kaddr, 0);
2007                 WARN_ON(err);
2008         }
2009
2010         if (err) {
2011                 kfree(map);
2012                 goto out;
2013         }
2014
2015         map->page_off = start_off;
2016         map->region = free_on_close ? reg : NULL;
2017         map->kctx = reg->kctx;
2018         map->vm_start = vma->vm_start + aligned_offset;
2019         if (aligned_offset) {
2020                 KBASE_DEBUG_ASSERT(!start_off);
2021                 map->vm_end = map->vm_start + (reg->nr_pages << PAGE_SHIFT);
2022         } else {
2023                 map->vm_end = vma->vm_end;
2024         }
2025         map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2026         map->count = 1; /* start with one ref */
2027
2028         if (reg->flags & KBASE_REG_CPU_CACHED)
2029                 map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
2030
2031         list_add(&map->mappings_list, &map->alloc->mappings);
2032
2033  out:
2034         return err;
2035 }
2036
2037 static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr)
2038 {
2039         struct kbase_va_region *new_reg;
2040         u32 nr_pages;
2041         size_t size;
2042         int err = 0;
2043         u32 *tb;
2044         int owns_tb = 1;
2045
2046         dev_dbg(kctx->kbdev->dev, "in %s\n", __func__);
2047         size = (vma->vm_end - vma->vm_start);
2048         nr_pages = size >> PAGE_SHIFT;
2049
2050         if (!kctx->jctx.tb) {
2051                 KBASE_DEBUG_ASSERT(0 != size);
2052                 tb = vmalloc_user(size);
2053
2054                 if (NULL == tb) {
2055                         err = -ENOMEM;
2056                         goto out;
2057                 }
2058
2059                 err = kbase_device_trace_buffer_install(kctx, tb, size);
2060                 if (err) {
2061                         vfree(tb);
2062                         goto out;
2063                 }
2064         } else {
2065                 err = -EINVAL;
2066                 goto out;
2067         }
2068
2069         *kaddr = kctx->jctx.tb;
2070
2071         new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2072         if (!new_reg) {
2073                 err = -ENOMEM;
2074                 WARN_ON(1);
2075                 goto out_no_region;
2076         }
2077
2078         new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
2079         if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2080                 err = -ENOMEM;
2081                 new_reg->cpu_alloc = NULL;
2082                 WARN_ON(1);
2083                 goto out_no_alloc;
2084         }
2085
2086         new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2087
2088         new_reg->cpu_alloc->imported.kctx = kctx;
2089         new_reg->flags &= ~KBASE_REG_FREE;
2090         new_reg->flags |= KBASE_REG_CPU_CACHED;
2091
2092         /* alloc now owns the tb */
2093         owns_tb = 0;
2094
2095         if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2096                 err = -ENOMEM;
2097                 WARN_ON(1);
2098                 goto out_no_va_region;
2099         }
2100
2101         *reg = new_reg;
2102
2103         /* map read only, noexec */
2104         vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2105         /* the rest of the flags is added by the cpu_mmap handler */
2106
2107         dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
2108         return 0;
2109
2110 out_no_va_region:
2111 out_no_alloc:
2112         kbase_free_alloced_region(new_reg);
2113 out_no_region:
2114         if (owns_tb) {
2115                 kbase_device_trace_buffer_uninstall(kctx);
2116                 vfree(tb);
2117         }
2118 out:
2119         return err;
2120 }
2121
2122 static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr)
2123 {
2124         struct kbase_va_region *new_reg;
2125         void *kaddr;
2126         u32 nr_pages;
2127         size_t size;
2128         int err = 0;
2129
2130         dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
2131         size = (vma->vm_end - vma->vm_start);
2132         nr_pages = size >> PAGE_SHIFT;
2133
2134         kaddr = kbase_mmu_dump(kctx, nr_pages);
2135
2136         if (!kaddr) {
2137                 err = -ENOMEM;
2138                 goto out;
2139         }
2140
2141         new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2142         if (!new_reg) {
2143                 err = -ENOMEM;
2144                 WARN_ON(1);
2145                 goto out;
2146         }
2147
2148         new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
2149         if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2150                 err = -ENOMEM;
2151                 new_reg->cpu_alloc = NULL;
2152                 WARN_ON(1);
2153                 goto out_no_alloc;
2154         }
2155
2156         new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2157
2158         new_reg->flags &= ~KBASE_REG_FREE;
2159         new_reg->flags |= KBASE_REG_CPU_CACHED;
2160         if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2161                 err = -ENOMEM;
2162                 WARN_ON(1);
2163                 goto out_va_region;
2164         }
2165
2166         *kmap_addr = kaddr;
2167         *reg = new_reg;
2168
2169         dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
2170         return 0;
2171
2172 out_no_alloc:
2173 out_va_region:
2174         kbase_free_alloced_region(new_reg);
2175 out:
2176         return err;
2177 }
2178
2179
2180 void kbase_os_mem_map_lock(struct kbase_context *kctx)
2181 {
2182         struct mm_struct *mm = current->mm;
2183         (void)kctx;
2184         down_read(&mm->mmap_sem);
2185 }
2186
2187 void kbase_os_mem_map_unlock(struct kbase_context *kctx)
2188 {
2189         struct mm_struct *mm = current->mm;
2190         (void)kctx;
2191         up_read(&mm->mmap_sem);
2192 }
2193
2194 int kbase_mmap(struct file *file, struct vm_area_struct *vma)
2195 {
2196         struct kbase_context *kctx = file->private_data;
2197         struct kbase_va_region *reg;
2198         void *kaddr = NULL;
2199         size_t nr_pages;
2200         int err = 0;
2201         int free_on_close = 0;
2202         struct device *dev = kctx->kbdev->dev;
2203         size_t aligned_offset = 0;
2204
2205         dev_dbg(dev, "kbase_mmap\n");
2206         nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2207
2208         /* strip away corresponding VM_MAY% flags to the VM_% flags requested */
2209         vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4);
2210
2211         if (0 == nr_pages) {
2212                 err = -EINVAL;
2213                 goto out;
2214         }
2215
2216         if (!(vma->vm_flags & VM_SHARED)) {
2217                 err = -EINVAL;
2218                 goto out;
2219         }
2220
2221         kbase_gpu_vm_lock(kctx);
2222
2223         if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) {
2224                 /* The non-mapped tracking helper page */
2225                 err = kbase_tracking_page_setup(kctx, vma);
2226                 goto out_unlock;
2227         }
2228
2229         /* if not the MTP, verify that the MTP has been mapped */
2230         rcu_read_lock();
2231         /* catches both when the special page isn't present or
2232          * when we've forked */
2233         if (rcu_dereference(kctx->process_mm) != current->mm) {
2234                 err = -EINVAL;
2235                 rcu_read_unlock();
2236                 goto out_unlock;
2237         }
2238         rcu_read_unlock();
2239
2240         switch (vma->vm_pgoff) {
2241         case PFN_DOWN(BASEP_MEM_INVALID_HANDLE):
2242         case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE):
2243                 /* Illegal handle for direct map */
2244                 err = -EINVAL;
2245                 goto out_unlock;
2246         case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE):
2247                 err = kbase_trace_buffer_mmap(kctx, vma, &reg, &kaddr);
2248                 if (0 != err)
2249                         goto out_unlock;
2250                 dev_dbg(dev, "kbase_trace_buffer_mmap ok\n");
2251                 /* free the region on munmap */
2252                 free_on_close = 1;
2253                 goto map;
2254         case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE):
2255                 /* MMU dump */
2256                 err = kbase_mmu_dump_mmap(kctx, vma, &reg, &kaddr);
2257                 if (0 != err)
2258                         goto out_unlock;
2259                 /* free the region on munmap */
2260                 free_on_close = 1;
2261                 goto map;
2262         case PFN_DOWN(BASE_MEM_COOKIE_BASE) ...
2263              PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: {
2264                 /* SAME_VA stuff, fetch the right region */
2265                 int gpu_pc_bits;
2266                 int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
2267
2268                 gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
2269                 reg = kctx->pending_regions[cookie];
2270                 if (!reg) {
2271                         err = -ENOMEM;
2272                         goto out_unlock;
2273                 }
2274
2275                 if (reg->flags & KBASE_REG_ALIGNED) {
2276                         /* nr_pages must be able to hold alignment pages
2277                          * plus actual pages */
2278                         unsigned long align = 1ULL << gpu_pc_bits;
2279                         unsigned long extra_pages = 3 * PFN_DOWN(align);
2280                         unsigned long aligned_addr;
2281                         unsigned long aligned_addr_end;
2282                         unsigned long nr_bytes = reg->nr_pages << PAGE_SHIFT;
2283
2284                         if (kctx->api_version < KBASE_API_VERSION(8, 5))
2285                                 /* Maintain compatibility with old userspace */
2286                                 extra_pages = PFN_DOWN(align);
2287
2288                         if (nr_pages != reg->nr_pages + extra_pages) {
2289                                 /* incorrect mmap size */
2290                                 /* leave the cookie for a potential
2291                                  * later mapping, or to be reclaimed
2292                                  * later when the context is freed */
2293                                 err = -ENOMEM;
2294                                 goto out_unlock;
2295                         }
2296
2297                         aligned_addr = ALIGN(vma->vm_start, align);
2298                         aligned_addr_end = aligned_addr + nr_bytes;
2299
2300                         if (kctx->api_version >= KBASE_API_VERSION(8, 5)) {
2301                                 if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
2302                                         /* Can't end at 4GB boundary */
2303                                         aligned_addr += 2 * align;
2304                                 } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
2305                                         /* Can't start at 4GB boundary */
2306                                         aligned_addr += align;
2307                                 }
2308                         }
2309
2310                         aligned_offset = aligned_addr - vma->vm_start;
2311                 } else if (reg->nr_pages != nr_pages) {
2312                         /* incorrect mmap size */
2313                         /* leave the cookie for a potential later
2314                          * mapping, or to be reclaimed later when the
2315                          * context is freed */
2316                         err = -ENOMEM;
2317                         goto out_unlock;
2318                 }
2319
2320                 if ((vma->vm_flags & VM_READ &&
2321                                         !(reg->flags & KBASE_REG_CPU_RD)) ||
2322                                 (vma->vm_flags & VM_WRITE &&
2323                                  !(reg->flags & KBASE_REG_CPU_WR))) {
2324                         /* VM flags inconsistent with region flags */
2325                         err = -EPERM;
2326                         dev_err(dev, "%s:%d inconsistent VM flags\n",
2327                                         __FILE__, __LINE__);
2328                         goto out_unlock;
2329                 }
2330
2331                 /* adjust down nr_pages to what we have physically */
2332                 nr_pages = kbase_reg_current_backed_size(reg);
2333
2334                 if (kbase_gpu_mmap(kctx, reg,
2335                                         vma->vm_start + aligned_offset,
2336                                         reg->nr_pages, 1) != 0) {
2337                         dev_err(dev, "%s:%d\n", __FILE__, __LINE__);
2338                         /* Unable to map in GPU space. */
2339                         WARN_ON(1);
2340                         err = -ENOMEM;
2341                         goto out_unlock;
2342                 }
2343
2344                 /* no need for the cookie anymore */
2345                 kctx->pending_regions[cookie] = NULL;
2346                 kctx->cookies |= (1UL << cookie);
2347
2348                 /*
2349                  * Overwrite the offset with the
2350                  * region start_pfn, so we effectively
2351                  * map from offset 0 in the region.
2352                  */
2353                 vma->vm_pgoff = reg->start_pfn;
2354
2355                 /* free the region on munmap */
2356                 free_on_close = 1;
2357                 goto map;
2358         }
2359         default: {
2360                 reg = kbase_region_tracker_find_region_enclosing_address(kctx, (u64)vma->vm_pgoff << PAGE_SHIFT);
2361
2362                 if (reg && !(reg->flags & KBASE_REG_FREE)) {
2363                         /* will this mapping overflow the size of the region? */
2364                         if (nr_pages > (reg->nr_pages - (vma->vm_pgoff - reg->start_pfn)))
2365                                 goto overflow;
2366
2367                         if ((vma->vm_flags & VM_READ &&
2368                              !(reg->flags & KBASE_REG_CPU_RD)) ||
2369                             (vma->vm_flags & VM_WRITE &&
2370                              !(reg->flags & KBASE_REG_CPU_WR))) {
2371                                 /* VM flags inconsistent with region flags */
2372                                 err = -EPERM;
2373                                 dev_err(dev, "%s:%d inconsistent VM flags\n",
2374                                         __FILE__, __LINE__);
2375                                 goto out_unlock;
2376                         }
2377
2378 #ifdef CONFIG_DMA_SHARED_BUFFER
2379                         if (reg->cpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM)
2380                                 goto dma_map;
2381 #endif /* CONFIG_DMA_SHARED_BUFFER */
2382
2383                         /* limit what we map to the amount currently backed */
2384                         if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
2385                                 if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
2386                                         nr_pages = 0;
2387                                 else
2388                                         nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn);
2389                         }
2390
2391                         goto map;
2392                 }
2393
2394 overflow:
2395                 err = -ENOMEM;
2396                 goto out_unlock;
2397         } /* default */
2398         } /* switch */
2399 map:
2400         err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close);
2401
2402         if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) {
2403                 /* MMU dump - userspace should now have a reference on
2404                  * the pages, so we can now free the kernel mapping */
2405                 vfree(kaddr);
2406         }
2407         goto out_unlock;
2408
2409 #ifdef CONFIG_DMA_SHARED_BUFFER
2410 dma_map:
2411         err = dma_buf_mmap(reg->cpu_alloc->imported.umm.dma_buf, vma, vma->vm_pgoff - reg->start_pfn);
2412 #endif /* CONFIG_DMA_SHARED_BUFFER */
2413 out_unlock:
2414         kbase_gpu_vm_unlock(kctx);
2415 out:
2416         if (err)
2417                 dev_err(dev, "mmap failed %d\n", err);
2418
2419         return err;
2420 }
2421
2422 KBASE_EXPORT_TEST_API(kbase_mmap);
2423
2424 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2425                       unsigned long prot_request, struct kbase_vmap_struct *map)
2426 {
2427         struct kbase_va_region *reg;
2428         unsigned long page_index;
2429         unsigned int offset = gpu_addr & ~PAGE_MASK;
2430         size_t page_count = PFN_UP(offset + size);
2431         phys_addr_t *page_array;
2432         struct page **pages;
2433         void *cpu_addr = NULL;
2434         pgprot_t prot;
2435         size_t i;
2436         bool sync_needed;
2437
2438         if (!size || !map)
2439                 return NULL;
2440
2441         /* check if page_count calculation will wrap */
2442         if (size > ((size_t)-1 / PAGE_SIZE))
2443                 return NULL;
2444
2445         kbase_gpu_vm_lock(kctx);
2446
2447         reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
2448         if (!reg || (reg->flags & KBASE_REG_FREE))
2449                 goto out_unlock;
2450
2451         page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn;
2452
2453         /* check if page_index + page_count will wrap */
2454         if (-1UL - page_count < page_index)
2455                 goto out_unlock;
2456
2457         if (page_index + page_count > kbase_reg_current_backed_size(reg))
2458                 goto out_unlock;
2459
2460         if (reg->flags & KBASE_REG_DONT_NEED)
2461                 goto out_unlock;
2462
2463         /* check access permissions can be satisfied
2464          * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */
2465         if ((reg->flags & prot_request) != prot_request)
2466                 goto out_unlock;
2467
2468         page_array = kbase_get_cpu_phy_pages(reg);
2469         if (!page_array)
2470                 goto out_unlock;
2471
2472         pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
2473         if (!pages)
2474                 goto out_unlock;
2475
2476         for (i = 0; i < page_count; i++)
2477                 pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i]));
2478
2479         prot = PAGE_KERNEL;
2480         if (!(reg->flags & KBASE_REG_CPU_CACHED)) {
2481                 /* Map uncached */
2482                 prot = pgprot_writecombine(prot);
2483         }
2484         /* Note: enforcing a RO prot_request onto prot is not done, since:
2485          * - CPU-arch-specific integration required
2486          * - kbase_vmap() requires no access checks to be made/enforced */
2487
2488         cpu_addr = vmap(pages, page_count, VM_MAP, prot);
2489
2490         kfree(pages);
2491
2492         if (!cpu_addr)
2493                 goto out_unlock;
2494
2495         map->gpu_addr = gpu_addr;
2496         map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2497         map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index];
2498         map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
2499         map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
2500         map->addr = (void *)((uintptr_t)cpu_addr + offset);
2501         map->size = size;
2502         map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0;
2503         sync_needed = map->is_cached;
2504
2505 #ifdef CONFIG_MALI_COH_KERN
2506         /* kernel can use coherent memory if supported */
2507         if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2508                 sync_needed = false;
2509 #endif
2510
2511         if (sync_needed) {
2512                 /* Sync first page */
2513                 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2514                 phys_addr_t cpu_pa = map->cpu_pages[0];
2515                 phys_addr_t gpu_pa = map->gpu_pages[0];
2516
2517                 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2518                                 KBASE_SYNC_TO_CPU);
2519
2520                 /* Sync middle pages (if any) */
2521                 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2522                         cpu_pa = map->cpu_pages[i];
2523                         gpu_pa = map->gpu_pages[i];
2524                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2525                                         KBASE_SYNC_TO_CPU);
2526                 }
2527
2528                 /* Sync last page (if any) */
2529                 if (page_count > 1) {
2530                         cpu_pa = map->cpu_pages[page_count - 1];
2531                         gpu_pa = map->gpu_pages[page_count - 1];
2532                         sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2533                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2534                                         KBASE_SYNC_TO_CPU);
2535                 }
2536         }
2537         kbase_gpu_vm_unlock(kctx);
2538
2539         return map->addr;
2540
2541 out_unlock:
2542         kbase_gpu_vm_unlock(kctx);
2543         return NULL;
2544 }
2545
2546 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2547                 struct kbase_vmap_struct *map)
2548 {
2549         /* 0 is specified for prot_request to indicate no access checks should
2550          * be made.
2551          *
2552          * As mentioned in kbase_vmap_prot() this means that a kernel-side
2553          * CPU-RO mapping is not enforced to allow this to work */
2554         return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map);
2555 }
2556 KBASE_EXPORT_TEST_API(kbase_vmap);
2557
2558 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
2559 {
2560         void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
2561         bool sync_needed = map->is_cached;
2562         vunmap(addr);
2563 #ifdef CONFIG_MALI_COH_KERN
2564         /* kernel can use coherent memory if supported */
2565         if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2566                 sync_needed = false;
2567 #endif
2568         if (sync_needed) {
2569                 off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
2570                 size_t size = map->size;
2571                 size_t page_count = PFN_UP(offset + size);
2572                 size_t i;
2573
2574                 /* Sync first page */
2575                 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2576                 phys_addr_t cpu_pa = map->cpu_pages[0];
2577                 phys_addr_t gpu_pa = map->gpu_pages[0];
2578
2579                 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2580                                 KBASE_SYNC_TO_DEVICE);
2581
2582                 /* Sync middle pages (if any) */
2583                 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2584                         cpu_pa = map->cpu_pages[i];
2585                         gpu_pa = map->gpu_pages[i];
2586                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2587                                         KBASE_SYNC_TO_DEVICE);
2588                 }
2589
2590                 /* Sync last page (if any) */
2591                 if (page_count > 1) {
2592                         cpu_pa = map->cpu_pages[page_count - 1];
2593                         gpu_pa = map->gpu_pages[page_count - 1];
2594                         sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2595                         kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2596                                         KBASE_SYNC_TO_DEVICE);
2597                 }
2598         }
2599         map->gpu_addr = 0;
2600         map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc);
2601         map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc);
2602         map->cpu_pages = NULL;
2603         map->gpu_pages = NULL;
2604         map->addr = NULL;
2605         map->size = 0;
2606         map->is_cached = false;
2607 }
2608 KBASE_EXPORT_TEST_API(kbase_vunmap);
2609
2610 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages)
2611 {
2612         struct mm_struct *mm;
2613
2614         rcu_read_lock();
2615         mm = rcu_dereference(kctx->process_mm);
2616         if (mm) {
2617                 atomic_add(pages, &kctx->nonmapped_pages);
2618 #ifdef SPLIT_RSS_COUNTING
2619                 add_mm_counter(mm, MM_FILEPAGES, pages);
2620 #else
2621                 spin_lock(&mm->page_table_lock);
2622                 add_mm_counter(mm, MM_FILEPAGES, pages);
2623                 spin_unlock(&mm->page_table_lock);
2624 #endif
2625         }
2626         rcu_read_unlock();
2627 }
2628
2629 static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx)
2630 {
2631         int pages;
2632         struct mm_struct *mm;
2633
2634         spin_lock(&kctx->mm_update_lock);
2635         mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock));
2636         if (!mm) {
2637                 spin_unlock(&kctx->mm_update_lock);
2638                 return;
2639         }
2640
2641         rcu_assign_pointer(kctx->process_mm, NULL);
2642         spin_unlock(&kctx->mm_update_lock);
2643         synchronize_rcu();
2644
2645         pages = atomic_xchg(&kctx->nonmapped_pages, 0);
2646 #ifdef SPLIT_RSS_COUNTING
2647         add_mm_counter(mm, MM_FILEPAGES, -pages);
2648 #else
2649         spin_lock(&mm->page_table_lock);
2650         add_mm_counter(mm, MM_FILEPAGES, -pages);
2651         spin_unlock(&mm->page_table_lock);
2652 #endif
2653 }
2654
2655 static void kbase_special_vm_close(struct vm_area_struct *vma)
2656 {
2657         struct kbase_context *kctx;
2658
2659         kctx = vma->vm_private_data;
2660         kbasep_os_process_page_usage_drain(kctx);
2661 }
2662
2663 static const struct vm_operations_struct kbase_vm_special_ops = {
2664         .close = kbase_special_vm_close,
2665 };
2666
2667 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma)
2668 {
2669         /* check that this is the only tracking page */
2670         spin_lock(&kctx->mm_update_lock);
2671         if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) {
2672                 spin_unlock(&kctx->mm_update_lock);
2673                 return -EFAULT;
2674         }
2675
2676         rcu_assign_pointer(kctx->process_mm, current->mm);
2677
2678         spin_unlock(&kctx->mm_update_lock);
2679
2680         /* no real access */
2681         vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2682 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
2683         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
2684 #else
2685         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
2686 #endif
2687         vma->vm_ops = &kbase_vm_special_ops;
2688         vma->vm_private_data = kctx;
2689
2690         return 0;
2691 }
2692 void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle)
2693 {
2694         int i;
2695         int res;
2696         void *va;
2697         dma_addr_t  dma_pa;
2698         struct kbase_va_region *reg;
2699         phys_addr_t *page_array;
2700 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2701         unsigned long attrs = DMA_ATTR_WRITE_COMBINE;
2702 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2703         DEFINE_DMA_ATTRS(attrs);
2704 #endif
2705
2706         u32 pages = ((size - 1) >> PAGE_SHIFT) + 1;
2707         u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR |
2708                     BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR;
2709
2710         KBASE_DEBUG_ASSERT(kctx != NULL);
2711         KBASE_DEBUG_ASSERT(0 != size);
2712         KBASE_DEBUG_ASSERT(0 != pages);
2713
2714         if (size == 0)
2715                 goto err;
2716
2717         /* All the alloc calls return zeroed memory */
2718 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2719         va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL,
2720                              attrs);
2721 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2722         dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2723         va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL,
2724                              &attrs);
2725 #else
2726         va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL);
2727 #endif
2728         if (!va)
2729                 goto err;
2730
2731         /* Store the state so we can free it later. */
2732         handle->cpu_va = va;
2733         handle->dma_pa = dma_pa;
2734         handle->size   = size;
2735
2736
2737         reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA);
2738         if (!reg)
2739                 goto no_reg;
2740
2741         reg->flags &= ~KBASE_REG_FREE;
2742         kbase_update_region_flags(kctx, reg, flags);
2743
2744         reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
2745         if (IS_ERR_OR_NULL(reg->cpu_alloc))
2746                 goto no_alloc;
2747
2748         reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2749
2750         page_array = kbase_get_cpu_phy_pages(reg);
2751
2752         for (i = 0; i < pages; i++)
2753                 page_array[i] = dma_pa + (i << PAGE_SHIFT);
2754
2755         reg->cpu_alloc->nents = pages;
2756
2757         kbase_gpu_vm_lock(kctx);
2758         res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1);
2759         kbase_gpu_vm_unlock(kctx);
2760         if (res)
2761                 goto no_mmap;
2762
2763         return va;
2764
2765 no_mmap:
2766         kbase_mem_phy_alloc_put(reg->cpu_alloc);
2767         kbase_mem_phy_alloc_put(reg->gpu_alloc);
2768 no_alloc:
2769         kfree(reg);
2770 no_reg:
2771 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2772         dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs);
2773 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2774         dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs);
2775 #else
2776         dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa);
2777 #endif
2778 err:
2779         return NULL;
2780 }
2781 KBASE_EXPORT_SYMBOL(kbase_va_alloc);
2782
2783 void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle)
2784 {
2785         struct kbase_va_region *reg;
2786         int err;
2787 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \
2788         (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
2789         DEFINE_DMA_ATTRS(attrs);
2790 #endif
2791
2792         KBASE_DEBUG_ASSERT(kctx != NULL);
2793         KBASE_DEBUG_ASSERT(handle->cpu_va != NULL);
2794
2795         kbase_gpu_vm_lock(kctx);
2796         reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va);
2797         KBASE_DEBUG_ASSERT(reg);
2798         err = kbase_gpu_munmap(kctx, reg);
2799         kbase_gpu_vm_unlock(kctx);
2800         KBASE_DEBUG_ASSERT(!err);
2801
2802         kbase_mem_phy_alloc_put(reg->cpu_alloc);
2803         kbase_mem_phy_alloc_put(reg->gpu_alloc);
2804         kfree(reg);
2805
2806 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
2807         dma_free_attrs(kctx->kbdev->dev, handle->size,
2808                        handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE);
2809 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2810         dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2811         dma_free_attrs(kctx->kbdev->dev, handle->size,
2812                         handle->cpu_va, handle->dma_pa, &attrs);
2813 #else
2814         dma_free_writecombine(kctx->kbdev->dev, handle->size,
2815                                 handle->cpu_va, handle->dma_pa);
2816 #endif
2817 }
2818 KBASE_EXPORT_SYMBOL(kbase_va_free);
2819