3 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_mem_linux.c
22 * Base kernel memory APIs, Linux implementation.
25 #include <linux/compat.h>
26 #include <linux/kernel.h>
27 #include <linux/bug.h>
29 #include <linux/mman.h>
31 #include <linux/version.h>
32 #include <linux/dma-mapping.h>
33 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
34 #include <linux/dma-attrs.h>
35 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) */
36 #ifdef CONFIG_DMA_SHARED_BUFFER
37 #include <linux/dma-buf.h>
38 #endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
39 #include <linux/shrinker.h>
41 #include <mali_kbase.h>
42 #include <mali_kbase_mem_linux.h>
43 #include <mali_kbase_config_defaults.h>
44 #include <mali_kbase_hwaccess_time.h>
45 #include <mali_kbase_tlstream.h>
47 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
48 static const struct vm_operations_struct kbase_vm_ops;
51 * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation
52 * @kctx: Context the region belongs to
53 * @reg: The GPU region
54 * @new_pages: The number of pages after the shrink
55 * @old_pages: The number of pages before the shrink
57 * Return: 0 on success, -errno on error.
59 * Shrink (or completely remove) all CPU mappings which reference the shrunk
60 * part of the allocation.
62 * Note: Caller must be holding the processes mmap_sem lock.
64 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
65 struct kbase_va_region *reg,
66 u64 new_pages, u64 old_pages);
69 * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
70 * @kctx: Context the region belongs to
71 * @reg: The GPU region or NULL if there isn't one
72 * @new_pages: The number of pages after the shrink
73 * @old_pages: The number of pages before the shrink
75 * Return: 0 on success, negative -errno on error
77 * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
78 * itself is unmodified as we still need to reserve the VA, only the page tables
79 * will be modified by this function.
81 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
82 struct kbase_va_region *reg,
83 u64 new_pages, u64 old_pages);
85 struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, u64 *gpu_va, u16 *va_alignment)
90 struct kbase_va_region *reg;
93 KBASE_DEBUG_ASSERT(kctx);
94 KBASE_DEBUG_ASSERT(flags);
95 KBASE_DEBUG_ASSERT(gpu_va);
96 KBASE_DEBUG_ASSERT(va_alignment);
98 dev = kctx->kbdev->dev;
99 *va_alignment = 0; /* no alignment by default */
100 *gpu_va = 0; /* return 0 on failure */
102 gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
103 cpu_va_bits = BITS_PER_LONG;
106 dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!");
110 if (va_pages > (U64_MAX / PAGE_SIZE))
111 /* 64-bit address range is the max */
114 #if defined(CONFIG_64BIT)
119 if (!kbase_check_alloc_flags(*flags)) {
121 "kbase_mem_alloc called with bad flags (%llx)",
122 (unsigned long long)*flags);
126 if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
127 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
128 dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
131 if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
132 !kbase_device_is_cpu_coherent(kctx->kbdev)) {
133 /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
134 *flags &= ~BASE_MEM_COHERENT_SYSTEM;
137 /* Limit GPU executable allocs to GPU PC size */
138 if ((*flags & BASE_MEM_PROT_GPU_EX) &&
139 (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT)))
142 /* find out which VA zone to use */
143 if (*flags & BASE_MEM_SAME_VA)
144 zone = KBASE_REG_ZONE_SAME_VA;
145 else if (*flags & BASE_MEM_PROT_GPU_EX)
146 zone = KBASE_REG_ZONE_EXEC;
148 zone = KBASE_REG_ZONE_CUSTOM_VA;
150 reg = kbase_alloc_free_region(kctx, 0, va_pages, zone);
152 dev_err(dev, "Failed to allocate free region");
156 kbase_update_region_flags(kctx, reg, *flags);
158 if (kbase_reg_prepare_native(reg, kctx) != 0) {
159 dev_err(dev, "Failed to prepare region");
163 if (*flags & BASE_MEM_GROW_ON_GPF)
164 reg->extent = extent;
168 if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) {
169 dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
170 (unsigned long long)commit_pages,
171 (unsigned long long)va_pages);
175 kbase_gpu_vm_lock(kctx);
177 /* mmap needed to setup VA? */
178 if (*flags & BASE_MEM_SAME_VA) {
179 unsigned long prot = PROT_NONE;
180 unsigned long va_size = va_pages << PAGE_SHIFT;
181 unsigned long va_map = va_size;
182 unsigned long cookie, cookie_nr;
183 unsigned long cpu_addr;
185 /* Bind to a cookie */
186 if (!kctx->cookies) {
187 dev_err(dev, "No cookies available for allocation!");
188 kbase_gpu_vm_unlock(kctx);
191 /* return a cookie */
192 cookie_nr = __ffs(kctx->cookies);
193 kctx->cookies &= ~(1UL << cookie_nr);
194 BUG_ON(kctx->pending_regions[cookie_nr]);
195 kctx->pending_regions[cookie_nr] = reg;
197 kbase_gpu_vm_unlock(kctx);
199 /* relocate to correct base */
200 cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE);
201 cookie <<= PAGE_SHIFT;
203 /* See if we must align memory due to GPU PC bits vs CPU VA */
204 if ((*flags & BASE_MEM_PROT_GPU_EX) &&
205 (cpu_va_bits > gpu_pc_bits)) {
206 *va_alignment = gpu_pc_bits;
207 reg->flags |= KBASE_REG_ALIGNED;
211 * Pre-10.1 UKU userland calls mmap for us so return the
212 * unaligned address and skip the map.
214 if (kctx->api_version < KBASE_API_VERSION(10, 1)) {
215 *gpu_va = (u64) cookie;
222 * We still need to return alignment for old userspace.
225 va_map += 3 * (1UL << *va_alignment);
227 if (*flags & BASE_MEM_PROT_CPU_RD)
229 if (*flags & BASE_MEM_PROT_CPU_WR)
232 cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot,
235 if (IS_ERR_VALUE(cpu_addr)) {
236 kctx->pending_regions[cookie_nr] = NULL;
237 kctx->cookies |= (1UL << cookie_nr);
242 * If we had to allocate extra VA space to force the
243 * alignment release it.
246 unsigned long alignment = 1UL << *va_alignment;
247 unsigned long align_mask = alignment - 1;
249 unsigned long addr_end;
250 unsigned long aligned_addr;
251 unsigned long aligned_addr_end;
254 addr_end = addr + va_map;
256 aligned_addr = (addr + align_mask) &
258 aligned_addr_end = aligned_addr + va_size;
260 if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
262 * Can't end at 4GB boundary on some GPUs as
263 * it will halt the shader.
265 aligned_addr += 2 * alignment;
266 aligned_addr_end += 2 * alignment;
267 } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
269 * Can't start at 4GB boundary on some GPUs as
270 * it will halt the shader.
272 aligned_addr += alignment;
273 aligned_addr_end += alignment;
276 /* anything to chop off at the start? */
277 if (addr != aligned_addr)
278 vm_munmap(addr, aligned_addr - addr);
280 /* anything at the end? */
281 if (addr_end != aligned_addr_end)
282 vm_munmap(aligned_addr_end,
283 addr_end - aligned_addr_end);
285 *gpu_va = (u64) aligned_addr;
287 *gpu_va = (u64) cpu_addr;
288 } else /* we control the VA */ {
289 if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) {
290 dev_warn(dev, "Failed to map memory on GPU");
291 kbase_gpu_vm_unlock(kctx);
294 /* return real GPU VA */
295 *gpu_va = reg->start_pfn << PAGE_SHIFT;
297 kbase_gpu_vm_unlock(kctx);
305 kbase_mem_phy_alloc_put(reg->cpu_alloc);
306 kbase_mem_phy_alloc_put(reg->gpu_alloc);
315 KBASE_EXPORT_TEST_API(kbase_mem_alloc);
317 int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out)
319 struct kbase_va_region *reg;
322 KBASE_DEBUG_ASSERT(kctx);
323 KBASE_DEBUG_ASSERT(out);
325 kbase_gpu_vm_lock(kctx);
327 /* Validate the region */
328 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
329 if (!reg || (reg->flags & KBASE_REG_FREE))
333 case KBASE_MEM_QUERY_COMMIT_SIZE:
334 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) {
335 *out = kbase_reg_current_backed_size(reg);
338 struct kbase_aliased *aliased;
340 aliased = reg->cpu_alloc->imported.alias.aliased;
341 for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++)
342 *out += aliased[i].length;
345 case KBASE_MEM_QUERY_VA_SIZE:
346 *out = reg->nr_pages;
348 case KBASE_MEM_QUERY_FLAGS:
351 if (KBASE_REG_CPU_WR & reg->flags)
352 *out |= BASE_MEM_PROT_CPU_WR;
353 if (KBASE_REG_CPU_RD & reg->flags)
354 *out |= BASE_MEM_PROT_CPU_RD;
355 if (KBASE_REG_CPU_CACHED & reg->flags)
356 *out |= BASE_MEM_CACHED_CPU;
357 if (KBASE_REG_GPU_WR & reg->flags)
358 *out |= BASE_MEM_PROT_GPU_WR;
359 if (KBASE_REG_GPU_RD & reg->flags)
360 *out |= BASE_MEM_PROT_GPU_RD;
361 if (!(KBASE_REG_GPU_NX & reg->flags))
362 *out |= BASE_MEM_PROT_GPU_EX;
363 if (KBASE_REG_SHARE_BOTH & reg->flags)
364 *out |= BASE_MEM_COHERENT_SYSTEM;
365 if (KBASE_REG_SHARE_IN & reg->flags)
366 *out |= BASE_MEM_COHERENT_LOCAL;
377 kbase_gpu_vm_unlock(kctx);
382 * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the
383 * Ephemeral memory eviction list.
385 * @sc: Shrinker control
387 * Return: Number of pages which can be freed.
390 unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s,
391 struct shrink_control *sc)
393 struct kbase_context *kctx;
394 struct kbase_mem_phy_alloc *alloc;
395 unsigned long pages = 0;
397 kctx = container_of(s, struct kbase_context, reclaim);
399 mutex_lock(&kctx->evict_lock);
401 list_for_each_entry(alloc, &kctx->evict_list, evict_node)
402 pages += alloc->nents;
404 mutex_unlock(&kctx->evict_lock);
409 * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction
410 * list for pages and try to reclaim them.
412 * @sc: Shrinker control
414 * Return: Number of pages freed (can be less then requested) or -1 if the
415 * shrinker failed to free pages in its pool.
418 * This function accesses region structures without taking the region lock,
419 * this is required as the OOM killer can call the shrinker after the region
420 * lock has already been held.
421 * This is safe as we can guarantee that a region on the eviction list will
422 * not be freed (kbase_mem_free_region removes the allocation from the list
423 * before destroying it), or modified by other parts of the driver.
424 * The eviction list itself is guarded by the eviction lock and the MMU updates
425 * are protected by their own lock.
428 unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
429 struct shrink_control *sc)
431 struct kbase_context *kctx;
432 struct kbase_mem_phy_alloc *alloc;
433 struct kbase_mem_phy_alloc *tmp;
434 unsigned long freed = 0;
436 kctx = container_of(s, struct kbase_context, reclaim);
437 mutex_lock(&kctx->evict_lock);
439 list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
442 err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
446 * Failed to remove GPU mapping, tell the shrinker
447 * to stop trying to shrink our slab even though we
455 * Update alloc->evicted before freeing the backing so the
456 * helper can determine that it needs to bypass the accounting
459 alloc->evicted = alloc->nents;
461 kbase_free_phy_pages_helper(alloc, alloc->evicted);
462 freed += alloc->evicted;
463 list_del_init(&alloc->evict_node);
466 * Inform the JIT allocator this region has lost backing
467 * as it might need to free the allocation.
469 kbase_jit_backing_lost(alloc->reg);
471 /* Enough pages have been freed so stop now */
472 if (freed > sc->nr_to_scan)
476 mutex_unlock(&kctx->evict_lock);
481 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
482 static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s,
483 struct shrink_control *sc)
485 if (sc->nr_to_scan == 0)
486 return kbase_mem_evictable_reclaim_count_objects(s, sc);
488 return kbase_mem_evictable_reclaim_scan_objects(s, sc);
492 int kbase_mem_evictable_init(struct kbase_context *kctx)
494 INIT_LIST_HEAD(&kctx->evict_list);
495 mutex_init(&kctx->evict_lock);
497 /* Register shrinker */
498 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
499 kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink;
501 kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects;
502 kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects;
504 kctx->reclaim.seeks = DEFAULT_SEEKS;
505 /* Kernel versions prior to 3.1 :
506 * struct shrinker does not define batch */
507 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
508 kctx->reclaim.batch = 0;
510 register_shrinker(&kctx->reclaim);
514 void kbase_mem_evictable_deinit(struct kbase_context *kctx)
516 unregister_shrinker(&kctx->reclaim);
519 struct kbase_mem_zone_cache_entry {
520 /* List head used to link the cache entry to the memory allocation. */
521 struct list_head zone_node;
522 /* The zone the cacheline is for. */
524 /* The number of pages in the allocation which belong to this zone. */
528 static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc,
531 struct kbase_mem_zone_cache_entry *cache = NULL;
535 for (i = start_offset; i < alloc->nents; i++) {
536 struct page *p = phys_to_page(alloc->pages[i]);
537 struct zone *zone = page_zone(p);
540 if (cache && (cache->zone == zone)) {
542 * Fast path check as most of the time adjacent
543 * pages come from the same zone.
548 * Slow path check, walk all the cache entries to see
549 * if we already know about this zone.
551 list_for_each_entry(cache, &alloc->zone_cache, zone_node) {
552 if (cache->zone == zone) {
559 /* This zone wasn't found in the cache, create an entry for it */
561 cache = kmalloc(sizeof(*cache), GFP_KERNEL);
568 list_add(&cache->zone_node, &alloc->zone_cache);
579 int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
583 * Bail if the zone cache is empty, only update the cache if it
584 * existed in the first place.
586 if (list_empty(&alloc->zone_cache))
589 return kbase_zone_cache_builder(alloc, start_offset);
592 int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc)
594 /* Bail if the zone cache already exists */
595 if (!list_empty(&alloc->zone_cache))
598 return kbase_zone_cache_builder(alloc, 0);
601 void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc)
603 struct kbase_mem_zone_cache_entry *walker;
605 while(!list_empty(&alloc->zone_cache)){
606 walker = list_first_entry(&alloc->zone_cache,
607 struct kbase_mem_zone_cache_entry, zone_node);
608 list_del(&walker->zone_node);
614 * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
615 * @alloc: The physical allocation
617 static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc)
619 struct kbase_context *kctx = alloc->imported.kctx;
620 struct kbase_mem_zone_cache_entry *zone_cache;
621 int __maybe_unused new_page_count;
624 /* Attempt to build a zone cache of tracking */
625 err = kbase_zone_cache_build(alloc);
627 /* Bulk update all the zones */
628 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
629 zone_page_state_add(zone_cache->count,
630 zone_cache->zone, NR_SLAB_RECLAIMABLE);
633 /* Fall-back to page by page updates */
636 for (i = 0; i < alloc->nents; i++) {
637 struct page *p = phys_to_page(alloc->pages[i]);
638 struct zone *zone = page_zone(p);
640 zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE);
644 kbase_process_page_usage_dec(kctx, alloc->nents);
645 new_page_count = kbase_atomic_sub_pages(alloc->nents,
647 kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
649 kbase_tlstream_aux_pagesalloc(
651 (u64)new_page_count);
655 * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable.
656 * @alloc: The physical allocation
659 void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc)
661 struct kbase_context *kctx = alloc->imported.kctx;
662 struct kbase_mem_zone_cache_entry *zone_cache;
663 int __maybe_unused new_page_count;
666 new_page_count = kbase_atomic_add_pages(alloc->nents,
668 kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
670 /* Increase mm counters so that the allocation is accounted for
671 * against the process and thus is visible to the OOM killer,
672 * then remove it from the reclaimable accounting. */
673 kbase_process_page_usage_inc(kctx, alloc->nents);
675 /* Attempt to build a zone cache of tracking */
676 err = kbase_zone_cache_build(alloc);
678 /* Bulk update all the zones */
679 list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
680 zone_page_state_add(-zone_cache->count,
681 zone_cache->zone, NR_SLAB_RECLAIMABLE);
684 /* Fall-back to page by page updates */
687 for (i = 0; i < alloc->nents; i++) {
688 struct page *p = phys_to_page(alloc->pages[i]);
689 struct zone *zone = page_zone(p);
691 zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE);
695 kbase_tlstream_aux_pagesalloc(
697 (u64)new_page_count);
700 int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc)
702 struct kbase_context *kctx = gpu_alloc->imported.kctx;
705 lockdep_assert_held(&kctx->reg_lock);
707 /* This alloction can't already be on a list. */
708 WARN_ON(!list_empty(&gpu_alloc->evict_node));
711 * Try to shrink the CPU mappings as required, if we fail then
712 * fail the process of making this allocation evictable.
714 err = kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg,
715 0, gpu_alloc->nents);
720 * Add the allocation to the eviction list, after this point the shrink
723 mutex_lock(&kctx->evict_lock);
724 list_add(&gpu_alloc->evict_node, &kctx->evict_list);
725 mutex_unlock(&kctx->evict_lock);
726 kbase_mem_evictable_mark_reclaim(gpu_alloc);
728 gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED;
732 bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc)
734 struct kbase_context *kctx = gpu_alloc->imported.kctx;
737 lockdep_assert_held(&kctx->reg_lock);
740 * First remove the allocation from the eviction list as it's no
741 * longer eligible for eviction.
743 mutex_lock(&kctx->evict_lock);
744 list_del_init(&gpu_alloc->evict_node);
745 mutex_unlock(&kctx->evict_lock);
747 if (gpu_alloc->evicted == 0) {
749 * The backing is still present, update the VM stats as it's
752 kbase_mem_evictable_unmark_reclaim(gpu_alloc);
754 /* If the region is still alive ... */
755 if (gpu_alloc->reg) {
756 /* ... allocate replacement backing ... */
757 err = kbase_alloc_phy_pages_helper(gpu_alloc,
761 * ... and grow the mapping back to its
765 err = kbase_mem_grow_gpu_mapping(kctx,
767 gpu_alloc->evicted, 0);
769 gpu_alloc->evicted = 0;
773 /* If the region is still alive remove the DONT_NEED attribute. */
775 gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED;
780 int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask)
782 struct kbase_va_region *reg;
784 unsigned int real_flags = 0;
785 unsigned int prev_flags = 0;
786 bool prev_needed, new_needed;
788 KBASE_DEBUG_ASSERT(kctx);
793 /* nuke other bits */
796 /* check for only supported flags */
797 if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE))
800 /* mask covers bits we don't support? */
801 if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE))
805 if (BASE_MEM_COHERENT_SYSTEM & flags)
806 real_flags |= KBASE_REG_SHARE_BOTH;
807 else if (BASE_MEM_COHERENT_LOCAL & flags)
808 real_flags |= KBASE_REG_SHARE_IN;
810 /* now we can lock down the context, and find the region */
811 down_write(¤t->mm->mmap_sem);
812 kbase_gpu_vm_lock(kctx);
814 /* Validate the region */
815 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
816 if (!reg || (reg->flags & KBASE_REG_FREE))
819 /* Is the region being transitioning between not needed and needed? */
820 prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED;
821 new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED;
822 if (prev_needed != new_needed) {
823 /* Aliased allocations can't be made ephemeral */
824 if (atomic_read(®->cpu_alloc->gpu_mappings) > 1)
828 /* Only native allocations can be marked not needed */
829 if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
833 ret = kbase_mem_evictable_make(reg->gpu_alloc);
837 kbase_mem_evictable_unmake(reg->gpu_alloc);
841 /* limit to imported memory */
842 if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
843 (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
847 if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) {
852 /* save for roll back */
853 prev_flags = reg->flags;
854 reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH);
855 reg->flags |= real_flags;
857 /* Currently supporting only imported memory */
858 switch (reg->gpu_alloc->type) {
860 case KBASE_MEM_TYPE_IMPORTED_UMP:
861 ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags);
864 #ifdef CONFIG_DMA_SHARED_BUFFER
865 case KBASE_MEM_TYPE_IMPORTED_UMM:
866 /* Future use will use the new flags, existing mapping will NOT be updated
867 * as memory should not be in use by the GPU when updating the flags.
870 WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count);
877 /* roll back on error, i.e. not UMP */
879 reg->flags = prev_flags;
882 kbase_gpu_vm_unlock(kctx);
883 up_write(¤t->mm->mmap_sem);
888 #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS)
891 static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags)
893 struct kbase_va_region *reg;
896 const ump_dd_physical_block_64 *block_array;
899 ump_alloc_flags ump_flags;
900 ump_alloc_flags cpu_flags;
901 ump_alloc_flags gpu_flags;
903 if (*flags & BASE_MEM_SECURE)
906 umph = ump_dd_from_secure_id(id);
907 if (UMP_DD_INVALID_MEMORY_HANDLE == umph)
910 ump_flags = ump_dd_allocation_flags_get(umph);
911 cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK;
912 gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) &
915 *va_pages = ump_dd_size_get_64(umph);
916 *va_pages >>= PAGE_SHIFT;
921 if (*va_pages > (U64_MAX / PAGE_SIZE))
922 /* 64-bit address range is the max */
925 if (*flags & BASE_MEM_SAME_VA)
926 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
928 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
933 /* we've got pages to map now, and support SAME_VA */
934 *flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
936 reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
937 if (IS_ERR_OR_NULL(reg->gpu_alloc))
940 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
942 reg->gpu_alloc->imported.ump_handle = umph;
944 reg->flags &= ~KBASE_REG_FREE;
945 reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */
946 reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */
948 /* Override import flags based on UMP flags */
949 *flags &= ~(BASE_MEM_CACHED_CPU);
950 *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR);
951 *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR);
953 if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
954 (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) {
955 reg->flags |= KBASE_REG_CPU_CACHED;
956 *flags |= BASE_MEM_CACHED_CPU;
959 if (cpu_flags & UMP_PROT_CPU_WR) {
960 reg->flags |= KBASE_REG_CPU_WR;
961 *flags |= BASE_MEM_PROT_CPU_WR;
964 if (cpu_flags & UMP_PROT_CPU_RD) {
965 reg->flags |= KBASE_REG_CPU_RD;
966 *flags |= BASE_MEM_PROT_CPU_RD;
969 if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) ==
970 (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR))
971 reg->flags |= KBASE_REG_GPU_CACHED;
973 if (gpu_flags & UMP_PROT_DEVICE_WR) {
974 reg->flags |= KBASE_REG_GPU_WR;
975 *flags |= BASE_MEM_PROT_GPU_WR;
978 if (gpu_flags & UMP_PROT_DEVICE_RD) {
979 reg->flags |= KBASE_REG_GPU_RD;
980 *flags |= BASE_MEM_PROT_GPU_RD;
983 /* ump phys block query */
984 ump_dd_phys_blocks_get_64(umph, &block_count, &block_array);
986 for (i = 0; i < block_count; i++) {
987 for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) {
988 reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
992 reg->gpu_alloc->nents = *va_pages;
1001 ump_dd_release(umph);
1006 #endif /* CONFIG_UMP */
1008 #ifdef CONFIG_DMA_SHARED_BUFFER
1009 static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags)
1011 struct kbase_va_region *reg;
1012 struct dma_buf *dma_buf;
1013 struct dma_buf_attachment *dma_attachment;
1014 bool shared_zone = false;
1016 dma_buf = dma_buf_get(fd);
1017 if (IS_ERR_OR_NULL(dma_buf))
1020 dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev);
1021 if (!dma_attachment)
1024 *va_pages = PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT;
1028 if (*va_pages > (U64_MAX / PAGE_SIZE))
1029 /* 64-bit address range is the max */
1032 /* ignore SAME_VA */
1033 *flags &= ~BASE_MEM_SAME_VA;
1035 if (*flags & BASE_MEM_IMPORT_SHARED)
1039 if (!kctx->is_compat) {
1041 * 64-bit tasks require us to reserve VA on the CPU that we use
1049 *flags |= BASE_MEM_NEED_MMAP;
1050 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
1052 reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
1058 reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
1059 if (IS_ERR_OR_NULL(reg->gpu_alloc))
1062 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1064 /* No pages to map yet */
1065 reg->gpu_alloc->nents = 0;
1067 reg->flags &= ~KBASE_REG_FREE;
1068 reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */
1069 reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */
1070 reg->flags |= KBASE_REG_GPU_CACHED;
1072 if (*flags & BASE_MEM_PROT_CPU_WR)
1073 reg->flags |= KBASE_REG_CPU_WR;
1075 if (*flags & BASE_MEM_PROT_CPU_RD)
1076 reg->flags |= KBASE_REG_CPU_RD;
1078 if (*flags & BASE_MEM_PROT_GPU_WR)
1079 reg->flags |= KBASE_REG_GPU_WR;
1081 if (*flags & BASE_MEM_PROT_GPU_RD)
1082 reg->flags |= KBASE_REG_GPU_RD;
1084 if (*flags & BASE_MEM_SECURE)
1085 reg->flags |= KBASE_REG_SECURE;
1087 /* no read or write permission given on import, only on run do we give the right permissions */
1089 reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM;
1090 reg->gpu_alloc->imported.umm.sgt = NULL;
1091 reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
1092 reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment;
1093 reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0;
1102 dma_buf_detach(dma_buf, dma_attachment);
1104 dma_buf_put(dma_buf);
1108 #endif /* CONFIG_DMA_SHARED_BUFFER */
1111 static struct kbase_va_region *kbase_mem_from_user_buffer(
1112 struct kbase_context *kctx, unsigned long address,
1113 unsigned long size, u64 *va_pages, u64 *flags)
1115 struct kbase_va_region *reg;
1117 int zone = KBASE_REG_ZONE_CUSTOM_VA;
1118 bool shared_zone = false;
1120 *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) -
1125 if (*va_pages > (UINT64_MAX / PAGE_SIZE))
1126 /* 64-bit address range is the max */
1129 /* SAME_VA generally not supported with imported memory (no known use cases) */
1130 *flags &= ~BASE_MEM_SAME_VA;
1132 if (*flags & BASE_MEM_IMPORT_SHARED)
1136 if (!kctx->is_compat) {
1138 * 64-bit tasks require us to reserve VA on the CPU that we use
1146 *flags |= BASE_MEM_NEED_MMAP;
1147 zone = KBASE_REG_ZONE_SAME_VA;
1150 reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone);
1155 reg->gpu_alloc = kbase_alloc_create(*va_pages,
1156 KBASE_MEM_TYPE_IMPORTED_USER_BUF);
1157 if (IS_ERR_OR_NULL(reg->gpu_alloc))
1160 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1162 reg->flags &= ~KBASE_REG_FREE;
1163 reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */
1164 reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */
1166 if (*flags & BASE_MEM_PROT_CPU_WR)
1167 reg->flags |= KBASE_REG_CPU_WR;
1169 if (*flags & BASE_MEM_PROT_CPU_RD)
1170 reg->flags |= KBASE_REG_CPU_RD;
1172 if (*flags & BASE_MEM_PROT_GPU_WR)
1173 reg->flags |= KBASE_REG_GPU_WR;
1175 if (*flags & BASE_MEM_PROT_GPU_RD)
1176 reg->flags |= KBASE_REG_GPU_RD;
1178 down_read(¤t->mm->mmap_sem);
1180 /* A sanity check that get_user_pages will work on the memory */
1181 /* (so the initial import fails on weird memory regions rather than */
1182 /* the job failing when we try to handle the external resources). */
1183 /* It doesn't take a reference to the pages (because the page list is NULL). */
1184 /* We can't really store the page list because that would involve */
1185 /* keeping the pages pinned - instead we pin/unpin around the job */
1186 /* (as part of the external resources handling code) */
1187 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
1188 faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
1189 reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1191 faulted_pages = get_user_pages(address, *va_pages,
1192 reg->flags & KBASE_REG_GPU_WR, 0, NULL, NULL);
1194 up_read(¤t->mm->mmap_sem);
1196 if (faulted_pages != *va_pages)
1197 goto fault_mismatch;
1199 reg->gpu_alloc->imported.user_buf.size = size;
1200 reg->gpu_alloc->imported.user_buf.address = address;
1201 reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages;
1202 reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages,
1203 sizeof(struct page *), GFP_KERNEL);
1204 reg->gpu_alloc->imported.user_buf.mm = current->mm;
1205 atomic_inc(¤t->mm->mm_count);
1207 if (!reg->gpu_alloc->imported.user_buf.pages)
1210 reg->gpu_alloc->nents = 0;
1217 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1227 u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride,
1228 u64 nents, struct base_mem_aliasing_info *ai,
1231 struct kbase_va_region *reg;
1236 KBASE_DEBUG_ASSERT(kctx);
1237 KBASE_DEBUG_ASSERT(flags);
1238 KBASE_DEBUG_ASSERT(ai);
1239 KBASE_DEBUG_ASSERT(num_pages);
1241 /* mask to only allowed flags */
1242 *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
1243 BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL |
1244 BASE_MEM_COHERENT_SYSTEM_REQUIRED);
1246 if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
1247 dev_warn(kctx->kbdev->dev,
1248 "kbase_mem_alias called with bad flags (%llx)",
1249 (unsigned long long)*flags);
1252 coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 ||
1253 (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0;
1261 if ((nents * stride) > (U64_MAX / PAGE_SIZE))
1262 /* 64-bit address range is the max */
1265 /* calculate the number of pages this alias will cover */
1266 *num_pages = nents * stride;
1269 if (!kctx->is_compat) {
1270 /* 64-bit tasks must MMAP anyway, but not expose this address to
1272 *flags |= BASE_MEM_NEED_MMAP;
1273 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1274 KBASE_REG_ZONE_SAME_VA);
1279 reg = kbase_alloc_free_region(kctx, 0, *num_pages,
1280 KBASE_REG_ZONE_CUSTOM_VA);
1286 /* zero-sized page array, as we don't need one/can support one */
1287 reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
1288 if (IS_ERR_OR_NULL(reg->gpu_alloc))
1291 reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
1293 kbase_update_region_flags(kctx, reg, *flags);
1295 reg->gpu_alloc->imported.alias.nents = nents;
1296 reg->gpu_alloc->imported.alias.stride = stride;
1297 reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents);
1298 if (!reg->gpu_alloc->imported.alias.aliased)
1299 goto no_aliased_array;
1301 kbase_gpu_vm_lock(kctx);
1303 /* validate and add src handles */
1304 for (i = 0; i < nents; i++) {
1305 if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) {
1306 if (ai[i].handle.basep.handle !=
1307 BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE)
1308 goto bad_handle; /* unsupported magic handle */
1310 goto bad_handle; /* must be > 0 */
1311 if (ai[i].length > stride)
1312 goto bad_handle; /* can't be larger than the
1314 reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1316 struct kbase_va_region *aliasing_reg;
1317 struct kbase_mem_phy_alloc *alloc;
1319 aliasing_reg = kbase_region_tracker_find_region_base_address(
1321 (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT);
1323 /* validate found region */
1325 goto bad_handle; /* Not found */
1326 if (aliasing_reg->flags & KBASE_REG_FREE)
1327 goto bad_handle; /* Free region */
1328 if (aliasing_reg->flags & KBASE_REG_DONT_NEED)
1329 goto bad_handle; /* Ephemeral region */
1330 if (!aliasing_reg->gpu_alloc)
1331 goto bad_handle; /* No alloc */
1332 if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
1333 goto bad_handle; /* Not a native alloc */
1334 if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0))
1336 /* Non-coherent memory cannot alias
1337 coherent memory, and vice versa.*/
1339 /* check size against stride */
1341 goto bad_handle; /* must be > 0 */
1342 if (ai[i].length > stride)
1343 goto bad_handle; /* can't be larger than the
1346 alloc = aliasing_reg->gpu_alloc;
1348 /* check against the alloc's size */
1349 if (ai[i].offset > alloc->nents)
1350 goto bad_handle; /* beyond end */
1351 if (ai[i].offset + ai[i].length > alloc->nents)
1352 goto bad_handle; /* beyond end */
1354 reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
1355 reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
1356 reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset;
1361 if (!kctx->is_compat) {
1362 /* Bind to a cookie */
1363 if (!kctx->cookies) {
1364 dev_err(kctx->kbdev->dev, "No cookies available for allocation!");
1367 /* return a cookie */
1368 gpu_va = __ffs(kctx->cookies);
1369 kctx->cookies &= ~(1UL << gpu_va);
1370 BUG_ON(kctx->pending_regions[gpu_va]);
1371 kctx->pending_regions[gpu_va] = reg;
1373 /* relocate to correct base */
1374 gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1375 gpu_va <<= PAGE_SHIFT;
1376 } else /* we control the VA */ {
1380 if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
1381 dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
1384 /* return real GPU VA */
1385 gpu_va = reg->start_pfn << PAGE_SHIFT;
1388 reg->flags &= ~KBASE_REG_FREE;
1389 reg->flags &= ~KBASE_REG_GROWABLE;
1391 kbase_gpu_vm_unlock(kctx);
1400 kbase_gpu_vm_unlock(kctx);
1402 kbase_mem_phy_alloc_put(reg->cpu_alloc);
1403 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1414 int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
1415 void __user *phandle, u64 *gpu_va, u64 *va_pages,
1418 struct kbase_va_region *reg;
1420 KBASE_DEBUG_ASSERT(kctx);
1421 KBASE_DEBUG_ASSERT(gpu_va);
1422 KBASE_DEBUG_ASSERT(va_pages);
1423 KBASE_DEBUG_ASSERT(flags);
1426 if (!kctx->is_compat)
1427 *flags |= BASE_MEM_SAME_VA;
1430 if (!kbase_check_import_flags(*flags)) {
1431 dev_warn(kctx->kbdev->dev,
1432 "kbase_mem_import called with bad flags (%llx)",
1433 (unsigned long long)*flags);
1439 case BASE_MEM_IMPORT_TYPE_UMP: {
1442 if (get_user(id, (ump_secure_id __user *)phandle))
1445 reg = kbase_mem_from_ump(kctx, id, va_pages, flags);
1448 #endif /* CONFIG_UMP */
1449 #ifdef CONFIG_DMA_SHARED_BUFFER
1450 case BASE_MEM_IMPORT_TYPE_UMM: {
1453 if (get_user(fd, (int __user *)phandle))
1456 reg = kbase_mem_from_umm(kctx, fd, va_pages, flags);
1459 #endif /* CONFIG_DMA_SHARED_BUFFER */
1460 case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
1461 struct base_mem_import_user_buffer user_buffer;
1464 if (copy_from_user(&user_buffer, phandle,
1465 sizeof(user_buffer))) {
1468 #ifdef CONFIG_COMPAT
1469 if (kctx->is_compat)
1470 uptr = compat_ptr(user_buffer.ptr.compat_value);
1473 uptr = user_buffer.ptr.value;
1475 reg = kbase_mem_from_user_buffer(kctx,
1476 (unsigned long)uptr, user_buffer.length,
1490 kbase_gpu_vm_lock(kctx);
1492 /* mmap needed to setup VA? */
1493 if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) {
1494 /* Bind to a cookie */
1497 /* return a cookie */
1498 *gpu_va = __ffs(kctx->cookies);
1499 kctx->cookies &= ~(1UL << *gpu_va);
1500 BUG_ON(kctx->pending_regions[*gpu_va]);
1501 kctx->pending_regions[*gpu_va] = reg;
1503 /* relocate to correct base */
1504 *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
1505 *gpu_va <<= PAGE_SHIFT;
1507 } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) {
1508 /* we control the VA, mmap now to the GPU */
1509 if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
1511 /* return real GPU VA */
1512 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1514 /* we control the VA, but nothing to mmap yet */
1515 if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0)
1517 /* return real GPU VA */
1518 *gpu_va = reg->start_pfn << PAGE_SHIFT;
1521 /* clear out private flags */
1522 *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1);
1524 kbase_gpu_vm_unlock(kctx);
1530 kbase_gpu_vm_unlock(kctx);
1531 kbase_mem_phy_alloc_put(reg->cpu_alloc);
1532 kbase_mem_phy_alloc_put(reg->gpu_alloc);
1543 static int zap_range_nolock(struct mm_struct *mm,
1544 const struct vm_operations_struct *vm_ops,
1545 unsigned long start, unsigned long end)
1547 struct vm_area_struct *vma;
1548 int err = -EINVAL; /* in case end < start */
1550 while (start < end) {
1551 unsigned long local_start;
1552 unsigned long local_end;
1554 vma = find_vma_intersection(mm, start, end);
1559 if (vma->vm_ops != vm_ops)
1562 local_start = vma->vm_start;
1564 if (start > local_start)
1565 local_start = start;
1567 local_end = vma->vm_end;
1569 if (end < local_end)
1572 err = zap_vma_ptes(vma, local_start, local_end - local_start);
1577 /* go to next vma, if any */
1578 start = vma->vm_end;
1584 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
1585 struct kbase_va_region *reg,
1586 u64 new_pages, u64 old_pages)
1588 phys_addr_t *phy_pages;
1589 u64 delta = new_pages - old_pages;
1592 lockdep_assert_held(&kctx->reg_lock);
1594 /* Map the new pages into the GPU */
1595 phy_pages = kbase_get_gpu_phy_pages(reg);
1596 ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages,
1597 phy_pages + old_pages, delta, reg->flags);
1602 static int kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx,
1603 struct kbase_va_region *reg,
1604 u64 new_pages, u64 old_pages)
1606 struct kbase_mem_phy_alloc *cpu_alloc = reg->cpu_alloc;
1607 struct kbase_cpu_mapping *mapping;
1610 lockdep_assert_held(&kctx->process_mm->mmap_sem);
1612 list_for_each_entry(mapping, &cpu_alloc->mappings, mappings_list) {
1613 unsigned long mapping_size;
1615 mapping_size = (mapping->vm_end - mapping->vm_start)
1618 /* is this mapping affected ?*/
1619 if ((mapping->page_off + mapping_size) > new_pages) {
1620 unsigned long first_bad = 0;
1622 if (new_pages > mapping->page_off)
1623 first_bad = new_pages - mapping->page_off;
1625 err = zap_range_nolock(current->mm,
1628 (first_bad << PAGE_SHIFT),
1632 "Failed to zap VA range (0x%lx - 0x%lx);\n",
1634 (first_bad << PAGE_SHIFT),
1638 /* The zap failed, give up and exit */
1650 static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
1651 struct kbase_va_region *reg,
1652 u64 new_pages, u64 old_pages)
1654 u64 delta = old_pages - new_pages;
1657 ret = kbase_mmu_teardown_pages(kctx,
1658 reg->start_pfn + new_pages, delta);
1663 int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
1668 struct kbase_va_region *reg;
1669 bool read_locked = false;
1671 KBASE_DEBUG_ASSERT(kctx);
1672 KBASE_DEBUG_ASSERT(failure_reason);
1673 KBASE_DEBUG_ASSERT(gpu_addr != 0);
1675 down_write(¤t->mm->mmap_sem);
1676 kbase_gpu_vm_lock(kctx);
1678 /* Validate the region */
1679 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1680 if (!reg || (reg->flags & KBASE_REG_FREE)) {
1681 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1685 KBASE_DEBUG_ASSERT(reg->cpu_alloc);
1686 KBASE_DEBUG_ASSERT(reg->gpu_alloc);
1688 if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
1689 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1693 if (0 == (reg->flags & KBASE_REG_GROWABLE)) {
1694 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1698 if (new_pages > reg->nr_pages) {
1699 /* Would overflow the VA region */
1700 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS;
1704 /* can't be mapped more than once on the GPU */
1705 if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) {
1706 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1709 /* can't grow regions which are ephemeral */
1710 if (reg->flags & KBASE_REG_DONT_NEED) {
1711 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
1715 if (new_pages == reg->gpu_alloc->nents) {
1721 old_pages = kbase_reg_current_backed_size(reg);
1722 if (new_pages > old_pages) {
1723 delta = new_pages - old_pages;
1726 * No update to the mm so downgrade the writer lock to a read
1727 * lock so other readers aren't blocked after this point.
1729 downgrade_write(¤t->mm->mmap_sem);
1732 /* Allocate some more pages */
1733 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
1734 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1737 if (reg->cpu_alloc != reg->gpu_alloc) {
1738 if (kbase_alloc_phy_pages_helper(
1739 reg->gpu_alloc, delta) != 0) {
1740 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1741 kbase_free_phy_pages_helper(reg->cpu_alloc,
1747 /* No update required for CPU mappings, that's done on fault. */
1749 /* Update GPU mapping. */
1750 res = kbase_mem_grow_gpu_mapping(kctx, reg,
1751 new_pages, old_pages);
1753 /* On error free the new pages */
1755 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1756 if (reg->cpu_alloc != reg->gpu_alloc)
1757 kbase_free_phy_pages_helper(reg->gpu_alloc,
1759 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1763 delta = old_pages - new_pages;
1765 /* Update all CPU mapping(s) */
1766 res = kbase_mem_shrink_cpu_mapping(kctx, reg,
1767 new_pages, old_pages);
1769 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1773 /* Update the GPU mapping */
1774 res = kbase_mem_shrink_gpu_mapping(kctx, reg,
1775 new_pages, old_pages);
1777 *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
1781 kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
1782 if (reg->cpu_alloc != reg->gpu_alloc)
1783 kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
1787 kbase_gpu_vm_unlock(kctx);
1789 up_read(¤t->mm->mmap_sem);
1791 up_write(¤t->mm->mmap_sem);
1796 static void kbase_cpu_vm_open(struct vm_area_struct *vma)
1798 struct kbase_cpu_mapping *map = vma->vm_private_data;
1800 KBASE_DEBUG_ASSERT(map);
1801 KBASE_DEBUG_ASSERT(map->count > 0);
1802 /* non-atomic as we're under Linux' mm lock */
1806 static void kbase_cpu_vm_close(struct vm_area_struct *vma)
1808 struct kbase_cpu_mapping *map = vma->vm_private_data;
1810 KBASE_DEBUG_ASSERT(map);
1811 KBASE_DEBUG_ASSERT(map->count > 0);
1813 /* non-atomic as we're under Linux' mm lock */
1817 KBASE_DEBUG_ASSERT(map->kctx);
1818 KBASE_DEBUG_ASSERT(map->alloc);
1820 kbase_gpu_vm_lock(map->kctx);
1823 KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) ==
1824 KBASE_REG_ZONE_SAME_VA);
1825 /* Avoid freeing memory on the process death which results in
1826 * GPU Page Fault. Memory will be freed in kbase_destroy_context
1828 if (!(current->flags & PF_EXITING))
1829 kbase_mem_free_region(map->kctx, map->region);
1832 list_del(&map->mappings_list);
1834 kbase_gpu_vm_unlock(map->kctx);
1836 kbase_mem_phy_alloc_put(map->alloc);
1840 KBASE_EXPORT_TEST_API(kbase_cpu_vm_close);
1843 static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1845 struct kbase_cpu_mapping *map = vma->vm_private_data;
1849 KBASE_DEBUG_ASSERT(map);
1850 KBASE_DEBUG_ASSERT(map->count > 0);
1851 KBASE_DEBUG_ASSERT(map->kctx);
1852 KBASE_DEBUG_ASSERT(map->alloc);
1854 /* we don't use vmf->pgoff as it's affected by our mmap with
1855 * offset being a GPU VA or a cookie */
1856 rel_pgoff = ((unsigned long)vmf->virtual_address - map->vm_start)
1859 kbase_gpu_vm_lock(map->kctx);
1860 if (map->page_off + rel_pgoff >= map->alloc->nents)
1861 goto locked_bad_fault;
1863 /* Fault on access to DONT_NEED regions */
1864 if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED))
1865 goto locked_bad_fault;
1867 /* insert all valid pages from the fault location */
1869 i < MIN((vma->vm_end - vma->vm_start) >> PAGE_SHIFT,
1870 map->alloc->nents - map->page_off); i++) {
1871 int ret = vm_insert_pfn(vma, map->vm_start + (i << PAGE_SHIFT),
1872 PFN_DOWN(map->alloc->pages[map->page_off + i]));
1873 if (ret < 0 && ret != -EBUSY)
1874 goto locked_bad_fault;
1877 kbase_gpu_vm_unlock(map->kctx);
1878 /* we resolved it, nothing for VM to do */
1879 return VM_FAULT_NOPAGE;
1882 kbase_gpu_vm_unlock(map->kctx);
1883 return VM_FAULT_SIGBUS;
1886 static const struct vm_operations_struct kbase_vm_ops = {
1887 .open = kbase_cpu_vm_open,
1888 .close = kbase_cpu_vm_close,
1889 .fault = kbase_cpu_vm_fault
1892 static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close)
1894 struct kbase_cpu_mapping *map;
1895 u64 start_off = vma->vm_pgoff - reg->start_pfn;
1896 phys_addr_t *page_array;
1900 map = kzalloc(sizeof(*map), GFP_KERNEL);
1909 * VM_DONTCOPY - don't make this mapping available in fork'ed processes
1910 * VM_DONTEXPAND - disable mremap on this region
1911 * VM_IO - disables paging
1912 * VM_DONTDUMP - Don't include in core dumps (3.7 only)
1913 * VM_MIXEDMAP - Support mixing struct page*s and raw pfns.
1914 * This is needed to support using the dedicated and
1915 * the OS based memory backends together.
1918 * This will need updating to propagate coherency flags
1922 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
1923 vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO;
1925 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
1927 vma->vm_ops = &kbase_vm_ops;
1928 vma->vm_private_data = map;
1930 page_array = kbase_get_cpu_phy_pages(reg);
1932 if (!(reg->flags & KBASE_REG_CPU_CACHED) &&
1933 (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) {
1934 /* We can't map vmalloc'd memory uncached.
1935 * Other memory will have been returned from
1936 * kbase_mem_pool which would be
1937 * suitable for mapping uncached.
1940 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1944 unsigned long addr = vma->vm_start + aligned_offset;
1946 vma->vm_flags |= VM_PFNMAP;
1947 for (i = 0; i < nr_pages; i++) {
1948 unsigned long pfn = PFN_DOWN(page_array[i + start_off]);
1950 err = vm_insert_pfn(vma, addr, pfn);
1957 WARN_ON(aligned_offset);
1958 /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */
1959 vma->vm_flags |= VM_MIXEDMAP;
1960 /* vmalloc remaping is easy... */
1961 err = remap_vmalloc_range(vma, kaddr, 0);
1970 map->page_off = start_off;
1971 map->region = free_on_close ? reg : NULL;
1972 map->kctx = reg->kctx;
1973 map->vm_start = vma->vm_start + aligned_offset;
1974 if (aligned_offset) {
1975 KBASE_DEBUG_ASSERT(!start_off);
1976 map->vm_end = map->vm_start + (reg->nr_pages << PAGE_SHIFT);
1978 map->vm_end = vma->vm_end;
1980 map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
1981 map->count = 1; /* start with one ref */
1983 if (reg->flags & KBASE_REG_CPU_CACHED)
1984 map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
1986 list_add(&map->mappings_list, &map->alloc->mappings);
1992 static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr)
1994 struct kbase_va_region *new_reg;
2001 dev_dbg(kctx->kbdev->dev, "in %s\n", __func__);
2002 size = (vma->vm_end - vma->vm_start);
2003 nr_pages = size >> PAGE_SHIFT;
2005 if (!kctx->jctx.tb) {
2006 KBASE_DEBUG_ASSERT(0 != size);
2007 tb = vmalloc_user(size);
2014 err = kbase_device_trace_buffer_install(kctx, tb, size);
2024 *kaddr = kctx->jctx.tb;
2026 new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2033 new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
2034 if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2036 new_reg->cpu_alloc = NULL;
2041 new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2043 new_reg->cpu_alloc->imported.kctx = kctx;
2044 new_reg->flags &= ~KBASE_REG_FREE;
2045 new_reg->flags |= KBASE_REG_CPU_CACHED;
2047 /* alloc now owns the tb */
2050 if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2053 goto out_no_va_region;
2058 /* map read only, noexec */
2059 vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2060 /* the rest of the flags is added by the cpu_mmap handler */
2062 dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
2067 kbase_free_alloced_region(new_reg);
2070 kbase_device_trace_buffer_uninstall(kctx);
2077 static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr)
2079 struct kbase_va_region *new_reg;
2085 dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
2086 size = (vma->vm_end - vma->vm_start);
2087 nr_pages = size >> PAGE_SHIFT;
2089 kaddr = kbase_mmu_dump(kctx, nr_pages);
2096 new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA);
2103 new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
2104 if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
2106 new_reg->cpu_alloc = NULL;
2111 new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
2113 new_reg->flags &= ~KBASE_REG_FREE;
2114 new_reg->flags |= KBASE_REG_CPU_CACHED;
2115 if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
2124 dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
2129 kbase_free_alloced_region(new_reg);
2135 void kbase_os_mem_map_lock(struct kbase_context *kctx)
2137 struct mm_struct *mm = current->mm;
2139 down_read(&mm->mmap_sem);
2142 void kbase_os_mem_map_unlock(struct kbase_context *kctx)
2144 struct mm_struct *mm = current->mm;
2146 up_read(&mm->mmap_sem);
2149 #if defined(CONFIG_DMA_SHARED_BUFFER) && defined(CONFIG_MALI_TRACE_TIMELINE)
2150 /* This section is required only for instrumentation. */
2152 static void kbase_dma_buf_vm_open(struct vm_area_struct *vma)
2154 struct kbase_cpu_mapping *map = vma->vm_private_data;
2156 KBASE_DEBUG_ASSERT(map);
2157 KBASE_DEBUG_ASSERT(map->count > 0);
2158 /* Non-atomic as we're under Linux's mm lock. */
2162 static void kbase_dma_buf_vm_close(struct vm_area_struct *vma)
2164 struct kbase_cpu_mapping *map = vma->vm_private_data;
2166 KBASE_DEBUG_ASSERT(map);
2167 KBASE_DEBUG_ASSERT(map->count > 0);
2169 /* Non-atomic as we're under Linux's mm lock. */
2173 KBASE_DEBUG_ASSERT(map->kctx);
2175 kbase_gpu_vm_lock(map->kctx);
2176 list_del(&map->mappings_list);
2177 kbase_gpu_vm_unlock(map->kctx);
2181 static const struct vm_operations_struct kbase_dma_mmap_ops = {
2182 .open = kbase_dma_buf_vm_open,
2183 .close = kbase_dma_buf_vm_close,
2185 #endif /* CONFIG_DMA_SHARED_BUFFER && CONFIG_MALI_TRACE_TIMELINE */
2187 int kbase_mmap(struct file *file, struct vm_area_struct *vma)
2189 struct kbase_context *kctx = file->private_data;
2190 struct kbase_va_region *reg;
2194 int free_on_close = 0;
2195 struct device *dev = kctx->kbdev->dev;
2196 size_t aligned_offset = 0;
2198 dev_dbg(dev, "kbase_mmap\n");
2199 nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2201 /* strip away corresponding VM_MAY% flags to the VM_% flags requested */
2202 vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4);
2204 if (0 == nr_pages) {
2209 if (!(vma->vm_flags & VM_SHARED)) {
2214 kbase_gpu_vm_lock(kctx);
2216 if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) {
2217 /* The non-mapped tracking helper page */
2218 err = kbase_tracking_page_setup(kctx, vma);
2222 /* if not the MTP, verify that the MTP has been mapped */
2224 /* catches both when the special page isn't present or
2225 * when we've forked */
2226 if (rcu_dereference(kctx->process_mm) != current->mm) {
2233 switch (vma->vm_pgoff) {
2234 case PFN_DOWN(BASEP_MEM_INVALID_HANDLE):
2235 case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE):
2236 /* Illegal handle for direct map */
2239 case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE):
2240 err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr);
2243 dev_dbg(dev, "kbase_trace_buffer_mmap ok\n");
2244 /* free the region on munmap */
2247 case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE):
2249 err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr);
2252 /* free the region on munmap */
2255 case PFN_DOWN(BASE_MEM_COOKIE_BASE) ...
2256 PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: {
2257 /* SAME_VA stuff, fetch the right region */
2259 int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
2261 gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size;
2262 reg = kctx->pending_regions[cookie];
2268 if (reg->flags & KBASE_REG_ALIGNED) {
2269 /* nr_pages must be able to hold alignment pages
2270 * plus actual pages */
2271 unsigned long align = 1ULL << gpu_pc_bits;
2272 unsigned long extra_pages = 3 * PFN_DOWN(align);
2273 unsigned long aligned_addr;
2274 unsigned long aligned_addr_end;
2275 unsigned long nr_bytes = reg->nr_pages << PAGE_SHIFT;
2277 if (kctx->api_version < KBASE_API_VERSION(8, 5))
2278 /* Maintain compatibility with old userspace */
2279 extra_pages = PFN_DOWN(align);
2281 if (nr_pages != reg->nr_pages + extra_pages) {
2282 /* incorrect mmap size */
2283 /* leave the cookie for a potential
2284 * later mapping, or to be reclaimed
2285 * later when the context is freed */
2290 aligned_addr = ALIGN(vma->vm_start, align);
2291 aligned_addr_end = aligned_addr + nr_bytes;
2293 if (kctx->api_version >= KBASE_API_VERSION(8, 5)) {
2294 if ((aligned_addr_end & BASE_MEM_MASK_4GB) == 0) {
2295 /* Can't end at 4GB boundary */
2296 aligned_addr += 2 * align;
2297 } else if ((aligned_addr & BASE_MEM_MASK_4GB) == 0) {
2298 /* Can't start at 4GB boundary */
2299 aligned_addr += align;
2303 aligned_offset = aligned_addr - vma->vm_start;
2304 } else if (reg->nr_pages != nr_pages) {
2305 /* incorrect mmap size */
2306 /* leave the cookie for a potential later
2307 * mapping, or to be reclaimed later when the
2308 * context is freed */
2313 if ((vma->vm_flags & VM_READ &&
2314 !(reg->flags & KBASE_REG_CPU_RD)) ||
2315 (vma->vm_flags & VM_WRITE &&
2316 !(reg->flags & KBASE_REG_CPU_WR))) {
2317 /* VM flags inconsistent with region flags */
2319 dev_err(dev, "%s:%d inconsistent VM flags\n",
2320 __FILE__, __LINE__);
2324 /* adjust down nr_pages to what we have physically */
2325 nr_pages = kbase_reg_current_backed_size(reg);
2327 if (kbase_gpu_mmap(kctx, reg,
2328 vma->vm_start + aligned_offset,
2329 reg->nr_pages, 1) != 0) {
2330 dev_err(dev, "%s:%d\n", __FILE__, __LINE__);
2331 /* Unable to map in GPU space. */
2337 /* no need for the cookie anymore */
2338 kctx->pending_regions[cookie] = NULL;
2339 kctx->cookies |= (1UL << cookie);
2342 * Overwrite the offset with the
2343 * region start_pfn, so we effectively
2344 * map from offset 0 in the region.
2346 vma->vm_pgoff = reg->start_pfn;
2348 /* free the region on munmap */
2353 reg = kbase_region_tracker_find_region_enclosing_address(kctx, (u64)vma->vm_pgoff << PAGE_SHIFT);
2355 if (reg && !(reg->flags & KBASE_REG_FREE)) {
2356 /* will this mapping overflow the size of the region? */
2357 if (nr_pages > (reg->nr_pages - (vma->vm_pgoff - reg->start_pfn)))
2360 if ((vma->vm_flags & VM_READ &&
2361 !(reg->flags & KBASE_REG_CPU_RD)) ||
2362 (vma->vm_flags & VM_WRITE &&
2363 !(reg->flags & KBASE_REG_CPU_WR))) {
2364 /* VM flags inconsistent with region flags */
2366 dev_err(dev, "%s:%d inconsistent VM flags\n",
2367 __FILE__, __LINE__);
2371 #ifdef CONFIG_DMA_SHARED_BUFFER
2372 if (reg->cpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM)
2374 #endif /* CONFIG_DMA_SHARED_BUFFER */
2376 /* limit what we map to the amount currently backed */
2377 if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
2378 if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
2381 nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn);
2393 err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close);
2395 if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) {
2396 /* MMU dump - userspace should now have a reference on
2397 * the pages, so we can now free the kernel mapping */
2402 #ifdef CONFIG_DMA_SHARED_BUFFER
2404 err = dma_buf_mmap(reg->cpu_alloc->imported.umm.dma_buf, vma, vma->vm_pgoff - reg->start_pfn);
2405 #if defined(CONFIG_MALI_TRACE_TIMELINE)
2406 /* This section is required only for instrumentation. */
2407 /* Add created mapping to imported region mapping list.
2408 * It is important to make it visible to dumping infrastructure.
2409 * Add mapping only if vm_ops structure is not used by memory owner. */
2410 WARN_ON(vma->vm_ops);
2411 WARN_ON(vma->vm_private_data);
2412 if (!err && !vma->vm_ops && !vma->vm_private_data) {
2413 struct kbase_cpu_mapping *map = kzalloc(
2418 map->kctx = reg->kctx;
2420 map->page_off = vma->vm_pgoff;
2421 map->vm_start = vma->vm_start;
2422 map->vm_end = vma->vm_end;
2423 map->count = 1; /* start with one ref */
2425 vma->vm_ops = &kbase_dma_mmap_ops;
2426 vma->vm_private_data = map;
2429 &map->mappings_list,
2430 ®->cpu_alloc->mappings);
2433 #endif /* CONFIG_MALI_TRACE_TIMELINE */
2434 #endif /* CONFIG_DMA_SHARED_BUFFER */
2436 kbase_gpu_vm_unlock(kctx);
2439 dev_err(dev, "mmap failed %d\n", err);
2444 KBASE_EXPORT_TEST_API(kbase_mmap);
2446 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2447 unsigned long prot_request, struct kbase_vmap_struct *map)
2449 struct kbase_va_region *reg;
2450 unsigned long page_index;
2451 unsigned int offset = gpu_addr & ~PAGE_MASK;
2452 size_t page_count = PFN_UP(offset + size);
2453 phys_addr_t *page_array;
2454 struct page **pages;
2455 void *cpu_addr = NULL;
2463 /* check if page_count calculation will wrap */
2464 if (size > ((size_t)-1 / PAGE_SIZE))
2467 kbase_gpu_vm_lock(kctx);
2469 reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
2470 if (!reg || (reg->flags & KBASE_REG_FREE))
2473 page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn;
2475 /* check if page_index + page_count will wrap */
2476 if (-1UL - page_count < page_index)
2479 if (page_index + page_count > kbase_reg_current_backed_size(reg))
2482 if (reg->flags & KBASE_REG_DONT_NEED)
2485 /* check access permissions can be satisfied
2486 * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */
2487 if ((reg->flags & prot_request) != prot_request)
2490 page_array = kbase_get_cpu_phy_pages(reg);
2494 pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
2498 for (i = 0; i < page_count; i++)
2499 pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i]));
2502 if (!(reg->flags & KBASE_REG_CPU_CACHED)) {
2504 prot = pgprot_writecombine(prot);
2506 /* Note: enforcing a RO prot_request onto prot is not done, since:
2507 * - CPU-arch-specific integration required
2508 * - kbase_vmap() requires no access checks to be made/enforced */
2510 cpu_addr = vmap(pages, page_count, VM_MAP, prot);
2517 map->gpu_addr = gpu_addr;
2518 map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2519 map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index];
2520 map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
2521 map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
2522 map->addr = (void *)((uintptr_t)cpu_addr + offset);
2524 map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0;
2525 sync_needed = map->is_cached;
2527 #ifdef CONFIG_MALI_COH_KERN
2528 /* kernel can use coherent memory if supported */
2529 if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2530 sync_needed = false;
2534 /* Sync first page */
2535 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2536 phys_addr_t cpu_pa = map->cpu_pages[0];
2537 phys_addr_t gpu_pa = map->gpu_pages[0];
2539 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2542 /* Sync middle pages (if any) */
2543 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2544 cpu_pa = map->cpu_pages[i];
2545 gpu_pa = map->gpu_pages[i];
2546 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2550 /* Sync last page (if any) */
2551 if (page_count > 1) {
2552 cpu_pa = map->cpu_pages[page_count - 1];
2553 gpu_pa = map->gpu_pages[page_count - 1];
2554 sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2555 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2559 kbase_gpu_vm_unlock(kctx);
2564 kbase_gpu_vm_unlock(kctx);
2568 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
2569 struct kbase_vmap_struct *map)
2571 /* 0 is specified for prot_request to indicate no access checks should
2574 * As mentioned in kbase_vmap_prot() this means that a kernel-side
2575 * CPU-RO mapping is not enforced to allow this to work */
2576 return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map);
2578 KBASE_EXPORT_TEST_API(kbase_vmap);
2580 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
2582 void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
2583 bool sync_needed = map->is_cached;
2585 #ifdef CONFIG_MALI_COH_KERN
2586 /* kernel can use coherent memory if supported */
2587 if (kctx->kbdev->system_coherency == COHERENCY_ACE)
2588 sync_needed = false;
2591 off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
2592 size_t size = map->size;
2593 size_t page_count = PFN_UP(offset + size);
2596 /* Sync first page */
2597 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
2598 phys_addr_t cpu_pa = map->cpu_pages[0];
2599 phys_addr_t gpu_pa = map->gpu_pages[0];
2601 kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
2602 KBASE_SYNC_TO_DEVICE);
2604 /* Sync middle pages (if any) */
2605 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
2606 cpu_pa = map->cpu_pages[i];
2607 gpu_pa = map->gpu_pages[i];
2608 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
2609 KBASE_SYNC_TO_DEVICE);
2612 /* Sync last page (if any) */
2613 if (page_count > 1) {
2614 cpu_pa = map->cpu_pages[page_count - 1];
2615 gpu_pa = map->gpu_pages[page_count - 1];
2616 sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
2617 kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
2618 KBASE_SYNC_TO_DEVICE);
2622 map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc);
2623 map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc);
2624 map->cpu_pages = NULL;
2625 map->gpu_pages = NULL;
2628 map->is_cached = false;
2630 KBASE_EXPORT_TEST_API(kbase_vunmap);
2632 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages)
2634 struct mm_struct *mm;
2637 mm = rcu_dereference(kctx->process_mm);
2639 atomic_add(pages, &kctx->nonmapped_pages);
2640 #ifdef SPLIT_RSS_COUNTING
2641 add_mm_counter(mm, MM_FILEPAGES, pages);
2643 spin_lock(&mm->page_table_lock);
2644 add_mm_counter(mm, MM_FILEPAGES, pages);
2645 spin_unlock(&mm->page_table_lock);
2651 static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx)
2654 struct mm_struct *mm;
2656 spin_lock(&kctx->mm_update_lock);
2657 mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock));
2659 spin_unlock(&kctx->mm_update_lock);
2663 rcu_assign_pointer(kctx->process_mm, NULL);
2664 spin_unlock(&kctx->mm_update_lock);
2667 pages = atomic_xchg(&kctx->nonmapped_pages, 0);
2668 #ifdef SPLIT_RSS_COUNTING
2669 add_mm_counter(mm, MM_FILEPAGES, -pages);
2671 spin_lock(&mm->page_table_lock);
2672 add_mm_counter(mm, MM_FILEPAGES, -pages);
2673 spin_unlock(&mm->page_table_lock);
2677 static void kbase_special_vm_close(struct vm_area_struct *vma)
2679 struct kbase_context *kctx;
2681 kctx = vma->vm_private_data;
2682 kbasep_os_process_page_usage_drain(kctx);
2685 static const struct vm_operations_struct kbase_vm_special_ops = {
2686 .close = kbase_special_vm_close,
2689 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma)
2691 /* check that this is the only tracking page */
2692 spin_lock(&kctx->mm_update_lock);
2693 if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) {
2694 spin_unlock(&kctx->mm_update_lock);
2698 rcu_assign_pointer(kctx->process_mm, current->mm);
2700 spin_unlock(&kctx->mm_update_lock);
2702 /* no real access */
2703 vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
2704 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
2705 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
2707 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO;
2709 vma->vm_ops = &kbase_vm_special_ops;
2710 vma->vm_private_data = kctx;
2714 void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle)
2720 struct kbase_va_region *reg;
2721 phys_addr_t *page_array;
2722 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2723 DEFINE_DMA_ATTRS(attrs);
2726 u32 pages = ((size - 1) >> PAGE_SHIFT) + 1;
2727 u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR |
2728 BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR;
2730 KBASE_DEBUG_ASSERT(kctx != NULL);
2731 KBASE_DEBUG_ASSERT(0 != size);
2732 KBASE_DEBUG_ASSERT(0 != pages);
2737 /* All the alloc calls return zeroed memory */
2738 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2739 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2740 va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, &attrs);
2742 va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL);
2747 /* Store the state so we can free it later. */
2748 handle->cpu_va = va;
2749 handle->dma_pa = dma_pa;
2750 handle->size = size;
2753 reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA);
2757 reg->flags &= ~KBASE_REG_FREE;
2758 kbase_update_region_flags(kctx, reg, flags);
2760 reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
2761 if (IS_ERR_OR_NULL(reg->cpu_alloc))
2764 reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
2766 page_array = kbase_get_cpu_phy_pages(reg);
2768 for (i = 0; i < pages; i++)
2769 page_array[i] = dma_pa + (i << PAGE_SHIFT);
2771 reg->cpu_alloc->nents = pages;
2773 kbase_gpu_vm_lock(kctx);
2774 res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1);
2775 kbase_gpu_vm_unlock(kctx);
2782 kbase_mem_phy_alloc_put(reg->cpu_alloc);
2783 kbase_mem_phy_alloc_put(reg->gpu_alloc);
2787 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2788 dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs);
2790 dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa);
2795 KBASE_EXPORT_SYMBOL(kbase_va_alloc);
2797 void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle)
2799 struct kbase_va_region *reg;
2801 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2802 DEFINE_DMA_ATTRS(attrs);
2805 KBASE_DEBUG_ASSERT(kctx != NULL);
2806 KBASE_DEBUG_ASSERT(handle->cpu_va != NULL);
2808 kbase_gpu_vm_lock(kctx);
2809 reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va);
2810 KBASE_DEBUG_ASSERT(reg);
2811 err = kbase_gpu_munmap(kctx, reg);
2812 kbase_gpu_vm_unlock(kctx);
2813 KBASE_DEBUG_ASSERT(!err);
2815 kbase_mem_phy_alloc_put(reg->cpu_alloc);
2816 kbase_mem_phy_alloc_put(reg->gpu_alloc);
2819 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
2820 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
2821 dma_free_attrs(kctx->kbdev->dev, handle->size,
2822 handle->cpu_va, handle->dma_pa, &attrs);
2824 dma_free_writecombine(kctx->kbdev->dev, handle->size,
2825 handle->cpu_va, handle->dma_pa);
2828 KBASE_EXPORT_SYMBOL(kbase_va_free);