3 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_mem.c
22 * Base kernel memory APIs
24 #ifdef CONFIG_DMA_SHARED_BUFFER
25 #include <linux/dma-buf.h>
26 #endif /* CONFIG_DMA_SHARED_BUFFER */
28 #include <linux/ump.h>
29 #endif /* CONFIG_UMP */
30 #include <linux/kernel.h>
31 #include <linux/bug.h>
32 #include <linux/compat.h>
34 #include <mali_kbase_config.h>
35 #include <mali_kbase.h>
36 #include <mali_midg_regmap.h>
37 #include <mali_kbase_cache_policy.h>
38 #include <mali_kbase_hw.h>
39 #include <mali_kbase_gator.h>
40 #include <mali_kbase_hwaccess_time.h>
41 #include <mali_kbase_tlstream.h>
44 * @brief Check the zone compatibility of two regions.
46 static int kbase_region_tracker_match_zone(struct kbase_va_region *reg1,
47 struct kbase_va_region *reg2)
49 return ((reg1->flags & KBASE_REG_ZONE_MASK) ==
50 (reg2->flags & KBASE_REG_ZONE_MASK));
53 KBASE_EXPORT_TEST_API(kbase_region_tracker_match_zone);
55 /* This function inserts a region into the tree. */
56 static void kbase_region_tracker_insert(struct kbase_context *kctx, struct kbase_va_region *new_reg)
58 u64 start_pfn = new_reg->start_pfn;
59 struct rb_node **link = &(kctx->reg_rbtree.rb_node);
60 struct rb_node *parent = NULL;
62 /* Find the right place in the tree using tree search */
64 struct kbase_va_region *old_reg;
67 old_reg = rb_entry(parent, struct kbase_va_region, rblink);
69 /* RBTree requires no duplicate entries. */
70 KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn);
72 if (old_reg->start_pfn > start_pfn)
73 link = &(*link)->rb_left;
75 link = &(*link)->rb_right;
78 /* Put the new node there, and rebalance tree */
79 rb_link_node(&(new_reg->rblink), parent, link);
80 rb_insert_color(&(new_reg->rblink), &(kctx->reg_rbtree));
83 /* Find allocated region enclosing free range. */
84 static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free(
85 struct kbase_context *kctx, u64 start_pfn, size_t nr_pages)
87 struct rb_node *rbnode;
88 struct kbase_va_region *reg;
90 u64 end_pfn = start_pfn + nr_pages;
92 rbnode = kctx->reg_rbtree.rb_node;
94 u64 tmp_start_pfn, tmp_end_pfn;
96 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
97 tmp_start_pfn = reg->start_pfn;
98 tmp_end_pfn = reg->start_pfn + reg->nr_pages;
100 /* If start is lower than this, go left. */
101 if (start_pfn < tmp_start_pfn)
102 rbnode = rbnode->rb_left;
103 /* If end is higher than this, then go right. */
104 else if (end_pfn > tmp_end_pfn)
105 rbnode = rbnode->rb_right;
113 /* Find region enclosing given address. */
114 struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr)
116 struct rb_node *rbnode;
117 struct kbase_va_region *reg;
118 u64 gpu_pfn = gpu_addr >> PAGE_SHIFT;
120 KBASE_DEBUG_ASSERT(NULL != kctx);
122 lockdep_assert_held(&kctx->reg_lock);
124 rbnode = kctx->reg_rbtree.rb_node;
126 u64 tmp_start_pfn, tmp_end_pfn;
128 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
129 tmp_start_pfn = reg->start_pfn;
130 tmp_end_pfn = reg->start_pfn + reg->nr_pages;
132 /* If start is lower than this, go left. */
133 if (gpu_pfn < tmp_start_pfn)
134 rbnode = rbnode->rb_left;
135 /* If end is higher than this, then go right. */
136 else if (gpu_pfn >= tmp_end_pfn)
137 rbnode = rbnode->rb_right;
145 KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address);
147 /* Find region with given base address */
148 struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr)
150 u64 gpu_pfn = gpu_addr >> PAGE_SHIFT;
151 struct rb_node *rbnode;
152 struct kbase_va_region *reg;
154 KBASE_DEBUG_ASSERT(NULL != kctx);
156 lockdep_assert_held(&kctx->reg_lock);
158 rbnode = kctx->reg_rbtree.rb_node;
160 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
161 if (reg->start_pfn > gpu_pfn)
162 rbnode = rbnode->rb_left;
163 else if (reg->start_pfn < gpu_pfn)
164 rbnode = rbnode->rb_right;
173 KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address);
175 /* Find region meeting given requirements */
176 static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align)
178 struct rb_node *rbnode;
179 struct kbase_va_region *reg;
181 /* Note that this search is a linear search, as we do not have a target
182 address in mind, so does not benefit from the rbtree search */
183 rbnode = rb_first(&(kctx->reg_rbtree));
185 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
186 if ((reg->nr_pages >= nr_pages) &&
187 (reg->flags & KBASE_REG_FREE) &&
188 kbase_region_tracker_match_zone(reg, reg_reqs)) {
189 /* Check alignment */
190 u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1);
192 if ((start_pfn >= reg->start_pfn) &&
193 (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) &&
194 ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1)))
197 rbnode = rb_next(rbnode);
204 * @brief Remove a region object from the global list.
206 * The region reg is removed, possibly by merging with other free and
207 * compatible adjacent regions. It must be called with the context
208 * region lock held. The associated memory is not released (see
209 * kbase_free_alloced_region). Internal use only.
211 static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg)
213 struct rb_node *rbprev;
214 struct kbase_va_region *prev = NULL;
215 struct rb_node *rbnext;
216 struct kbase_va_region *next = NULL;
218 int merged_front = 0;
222 /* Try to merge with the previous block first */
223 rbprev = rb_prev(&(reg->rblink));
225 prev = rb_entry(rbprev, struct kbase_va_region, rblink);
226 if ((prev->flags & KBASE_REG_FREE) && kbase_region_tracker_match_zone(prev, reg)) {
227 /* We're compatible with the previous VMA, merge with it */
228 prev->nr_pages += reg->nr_pages;
229 rb_erase(&(reg->rblink), &kctx->reg_rbtree);
235 /* Try to merge with the next block second */
236 /* Note we do the lookup here as the tree may have been rebalanced. */
237 rbnext = rb_next(&(reg->rblink));
239 /* We're compatible with the next VMA, merge with it */
240 next = rb_entry(rbnext, struct kbase_va_region, rblink);
241 if ((next->flags & KBASE_REG_FREE) && kbase_region_tracker_match_zone(next, reg)) {
242 next->start_pfn = reg->start_pfn;
243 next->nr_pages += reg->nr_pages;
244 rb_erase(&(reg->rblink), &kctx->reg_rbtree);
247 /* We already merged with prev, free it */
248 kbase_free_alloced_region(reg);
253 /* If we failed to merge then we need to add a new block */
254 if (!(merged_front || merged_back)) {
256 * We didn't merge anything. Add a new free
257 * placeholder and remove the original one.
259 struct kbase_va_region *free_reg;
261 free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK);
267 rb_replace_node(&(reg->rblink), &(free_reg->rblink), &(kctx->reg_rbtree));
274 KBASE_EXPORT_TEST_API(kbase_remove_va_region);
277 * @brief Insert a VA region to the list, replacing the current at_reg.
279 static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages)
283 /* Must be a free region */
284 KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0);
285 /* start_pfn should be contained within at_reg */
286 KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages));
287 /* at least nr_pages from start_pfn should be contained within at_reg */
288 KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages);
290 new_reg->start_pfn = start_pfn;
291 new_reg->nr_pages = nr_pages;
293 /* Regions are a whole use, so swap and delete old one. */
294 if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) {
295 rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), &(kctx->reg_rbtree));
296 kbase_free_alloced_region(at_reg);
298 /* New region replaces the start of the old one, so insert before. */
299 else if (at_reg->start_pfn == start_pfn) {
300 at_reg->start_pfn += nr_pages;
301 KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages);
302 at_reg->nr_pages -= nr_pages;
304 kbase_region_tracker_insert(kctx, new_reg);
306 /* New region replaces the end of the old one, so insert after. */
307 else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) {
308 at_reg->nr_pages -= nr_pages;
310 kbase_region_tracker_insert(kctx, new_reg);
312 /* New region splits the old one, so insert and create new */
314 struct kbase_va_region *new_front_reg;
316 new_front_reg = kbase_alloc_free_region(kctx,
318 start_pfn - at_reg->start_pfn,
319 at_reg->flags & KBASE_REG_ZONE_MASK);
322 at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages;
323 at_reg->start_pfn = start_pfn + nr_pages;
325 kbase_region_tracker_insert(kctx, new_front_reg);
326 kbase_region_tracker_insert(kctx, new_reg);
336 * @brief Add a VA region to the list.
338 int kbase_add_va_region(struct kbase_context *kctx,
339 struct kbase_va_region *reg, u64 addr,
340 size_t nr_pages, size_t align)
342 struct kbase_va_region *tmp;
343 u64 gpu_pfn = addr >> PAGE_SHIFT;
346 KBASE_DEBUG_ASSERT(NULL != kctx);
347 KBASE_DEBUG_ASSERT(NULL != reg);
349 lockdep_assert_held(&kctx->reg_lock);
354 /* must be a power of 2 */
355 KBASE_DEBUG_ASSERT((align & (align - 1)) == 0);
356 KBASE_DEBUG_ASSERT(nr_pages > 0);
358 /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */
360 struct device *dev = kctx->kbdev->dev;
362 KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1)));
364 tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages);
366 dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages);
371 if ((!kbase_region_tracker_match_zone(tmp, reg)) ||
372 (!(tmp->flags & KBASE_REG_FREE))) {
373 dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK);
374 dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages);
375 dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align);
380 err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages);
382 dev_warn(dev, "Failed to insert va region");
390 /* Path 2: Map any free address which meets the requirements. */
395 * Depending on the zone the allocation request is for
396 * we might need to retry it.
399 tmp = kbase_region_tracker_find_region_meeting_reqs(
400 kctx, reg, nr_pages, align);
402 start_pfn = (tmp->start_pfn + align - 1) &
404 err = kbase_insert_va_region_nolock(kctx, reg,
405 tmp, start_pfn, nr_pages);
410 * If the allocation is not from the same zone as JIT
411 * then don't retry, we're out of VA and there is
412 * nothing which can be done about it.
414 if ((reg->flags & KBASE_REG_ZONE_MASK) !=
415 KBASE_REG_ZONE_CUSTOM_VA)
417 } while (kbase_jit_evict(kctx));
427 KBASE_EXPORT_TEST_API(kbase_add_va_region);
430 * @brief Initialize the internal region tracker data structure.
432 static void kbase_region_tracker_ds_init(struct kbase_context *kctx,
433 struct kbase_va_region *same_va_reg,
434 struct kbase_va_region *exec_reg,
435 struct kbase_va_region *custom_va_reg)
437 kctx->reg_rbtree = RB_ROOT;
438 kbase_region_tracker_insert(kctx, same_va_reg);
440 /* exec and custom_va_reg doesn't always exist */
441 if (exec_reg && custom_va_reg) {
442 kbase_region_tracker_insert(kctx, exec_reg);
443 kbase_region_tracker_insert(kctx, custom_va_reg);
447 void kbase_region_tracker_term(struct kbase_context *kctx)
449 struct rb_node *rbnode;
450 struct kbase_va_region *reg;
453 rbnode = rb_first(&(kctx->reg_rbtree));
455 rb_erase(rbnode, &(kctx->reg_rbtree));
456 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
457 kbase_free_alloced_region(reg);
463 * Initialize the region tracker data structure.
465 int kbase_region_tracker_init(struct kbase_context *kctx)
467 struct kbase_va_region *same_va_reg;
468 struct kbase_va_region *exec_reg = NULL;
469 struct kbase_va_region *custom_va_reg = NULL;
470 size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE;
471 u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE;
472 u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT;
476 /* Take the lock as kbase_free_alloced_region requires it */
477 kbase_gpu_vm_lock(kctx);
479 #if defined(CONFIG_ARM64)
480 same_va_bits = VA_BITS;
481 #elif defined(CONFIG_X86_64)
483 #elif defined(CONFIG_64BIT)
484 #error Unsupported 64-bit architecture
490 else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA))
494 if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) {
499 same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1;
500 /* all have SAME_VA */
501 same_va_reg = kbase_alloc_free_region(kctx, 1,
503 KBASE_REG_ZONE_SAME_VA);
511 /* 32-bit clients have exec and custom VA zones */
512 if (kctx->is_compat) {
514 if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) {
516 goto fail_free_same_va;
518 /* If the current size of TMEM is out of range of the
519 * virtual address space addressable by the MMU then
520 * we should shrink it to fit
522 if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit)
523 custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE;
525 exec_reg = kbase_alloc_free_region(kctx,
526 KBASE_REG_ZONE_EXEC_BASE,
527 KBASE_REG_ZONE_EXEC_SIZE,
528 KBASE_REG_ZONE_EXEC);
532 goto fail_free_same_va;
535 custom_va_reg = kbase_alloc_free_region(kctx,
536 KBASE_REG_ZONE_CUSTOM_VA_BASE,
537 custom_va_size, KBASE_REG_ZONE_CUSTOM_VA);
539 if (!custom_va_reg) {
547 kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg);
549 kctx->same_va_end = same_va_pages + 1;
551 kbase_gpu_vm_unlock(kctx);
555 kbase_free_alloced_region(exec_reg);
557 kbase_free_alloced_region(same_va_reg);
559 kbase_gpu_vm_unlock(kctx);
563 int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages)
566 struct kbase_va_region *same_va;
567 struct kbase_va_region *custom_va_reg;
573 * Nothing to do for 32-bit clients, JIT uses the existing
579 #if defined(CONFIG_ARM64)
580 same_va_bits = VA_BITS;
581 #elif defined(CONFIG_X86_64)
583 #elif defined(CONFIG_64BIT)
584 #error Unsupported 64-bit architecture
587 if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA))
590 total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1;
592 kbase_gpu_vm_lock(kctx);
595 * Modify the same VA free region after creation. Be careful to ensure
596 * that allocations haven't been made as they could cause an overlap
597 * to happen with existing same VA allocations and the custom VA zone.
599 same_va = kbase_region_tracker_find_region_base_address(kctx,
606 /* The region flag or region size has changed since creation so bail. */
607 if ((!(same_va->flags & KBASE_REG_FREE)) ||
608 (same_va->nr_pages != total_va_size)) {
613 /* It's safe to adjust the same VA zone now */
614 same_va->nr_pages -= jit_va_pages;
615 kctx->same_va_end -= jit_va_pages;
618 * Create a custom VA zone at the end of the VA for allocations which
619 * JIT can use so it doesn't have to allocate VA from the kernel.
621 custom_va_reg = kbase_alloc_free_region(kctx,
624 KBASE_REG_ZONE_CUSTOM_VA);
625 if (!custom_va_reg) {
627 * The context will be destroyed if we fail here so no point
628 * reverting the change we made to same_va.
634 kbase_region_tracker_insert(kctx, custom_va_reg);
636 kbase_gpu_vm_unlock(kctx);
640 kbase_gpu_vm_unlock(kctx);
647 int kbase_mem_init(struct kbase_device *kbdev)
649 struct kbasep_mem_device *memdev;
651 KBASE_DEBUG_ASSERT(kbdev);
653 memdev = &kbdev->memdev;
654 kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX;
656 /* Initialize memory usage */
657 atomic_set(&memdev->used_pages, 0);
659 return kbase_mem_pool_init(&kbdev->mem_pool,
660 KBASE_MEM_POOL_MAX_SIZE_KBDEV, kbdev, NULL);
663 void kbase_mem_halt(struct kbase_device *kbdev)
668 void kbase_mem_term(struct kbase_device *kbdev)
670 struct kbasep_mem_device *memdev;
673 KBASE_DEBUG_ASSERT(kbdev);
675 memdev = &kbdev->memdev;
677 pages = atomic_read(&memdev->used_pages);
679 dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
681 kbase_mem_pool_term(&kbdev->mem_pool);
684 KBASE_EXPORT_TEST_API(kbase_mem_term);
690 * @brief Allocate a free region object.
692 * The allocated object is not part of any list yet, and is flagged as
693 * KBASE_REG_FREE. No mapping is allocated yet.
695 * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC
698 struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone)
700 struct kbase_va_region *new_reg;
702 KBASE_DEBUG_ASSERT(kctx != NULL);
704 /* zone argument should only contain zone related region flags */
705 KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0);
706 KBASE_DEBUG_ASSERT(nr_pages > 0);
707 /* 64-bit address range is the max */
708 KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE));
710 new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL);
715 new_reg->cpu_alloc = NULL; /* no alloc bound yet */
716 new_reg->gpu_alloc = NULL; /* no alloc bound yet */
717 new_reg->kctx = kctx;
718 new_reg->flags = zone | KBASE_REG_FREE;
720 new_reg->flags |= KBASE_REG_GROWABLE;
722 new_reg->start_pfn = start_pfn;
723 new_reg->nr_pages = nr_pages;
728 KBASE_EXPORT_TEST_API(kbase_alloc_free_region);
731 * @brief Free a region object.
733 * The described region must be freed of any mapping.
735 * If the region is not flagged as KBASE_REG_FREE, the region's
736 * alloc object will be released.
737 * It is a bug if no alloc object exists for non-free regions.
740 void kbase_free_alloced_region(struct kbase_va_region *reg)
742 if (!(reg->flags & KBASE_REG_FREE)) {
744 * The physical allocation should have been removed from the
745 * eviction list before this function is called. However, in the
746 * case of abnormal process termination or the app leaking the
747 * memory kbase_mem_free_region is not called so it can still be
748 * on the list at termination time of the region tracker.
750 if (!list_empty(®->gpu_alloc->evict_node)) {
752 * Unlink the physical allocation before unmaking it
753 * evictable so that the allocation isn't grown back to
754 * its last backed size as we're going to unmap it
757 reg->cpu_alloc->reg = NULL;
758 if (reg->cpu_alloc != reg->gpu_alloc)
759 reg->gpu_alloc->reg = NULL;
762 * If a region has been made evictable then we must
763 * unmake it before trying to free it.
764 * If the memory hasn't been reclaimed it will be
765 * unmapped and freed below, if it has been reclaimed
766 * then the operations below are no-ops.
768 if (reg->flags & KBASE_REG_DONT_NEED) {
769 KBASE_DEBUG_ASSERT(reg->cpu_alloc->type ==
770 KBASE_MEM_TYPE_NATIVE);
771 kbase_mem_evictable_unmake(reg->gpu_alloc);
776 * Remove the region from the sticky resource metadata
777 * list should it be there.
779 kbase_sticky_resource_release(reg->kctx, NULL,
780 reg->start_pfn << PAGE_SHIFT, true);
782 kbase_mem_phy_alloc_put(reg->cpu_alloc);
783 kbase_mem_phy_alloc_put(reg->gpu_alloc);
784 /* To detect use-after-free in debug builds */
785 KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE);
790 KBASE_EXPORT_TEST_API(kbase_free_alloced_region);
792 void kbase_mmu_update(struct kbase_context *kctx)
794 KBASE_DEBUG_ASSERT(NULL != kctx);
795 lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
796 /* ASSERT that the context has a valid as_nr, which is only the case
797 * when it's scheduled in.
799 * as_nr won't change because the caller has the runpool_irq lock */
800 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
801 lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex);
803 kctx->kbdev->mmu_mode->update(kctx);
806 KBASE_EXPORT_TEST_API(kbase_mmu_update);
808 void kbase_mmu_disable(struct kbase_context *kctx)
810 KBASE_DEBUG_ASSERT(NULL != kctx);
811 /* ASSERT that the context has a valid as_nr, which is only the case
812 * when it's scheduled in.
814 * as_nr won't change because the caller has the runpool_irq lock */
815 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
817 kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr);
820 KBASE_EXPORT_TEST_API(kbase_mmu_disable);
822 void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr)
824 kbdev->mmu_mode->disable_as(kbdev, as_nr);
827 int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align)
832 unsigned long mask = ~KBASE_REG_MEMATTR_MASK;
834 if ((kctx->kbdev->system_coherency == COHERENCY_ACE) &&
835 (reg->flags & KBASE_REG_SHARE_BOTH))
836 attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA);
838 attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC);
840 KBASE_DEBUG_ASSERT(NULL != kctx);
841 KBASE_DEBUG_ASSERT(NULL != reg);
843 err = kbase_add_va_region(kctx, reg, addr, nr_pages, align);
847 if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
849 struct kbase_mem_phy_alloc *alloc;
851 alloc = reg->gpu_alloc;
852 stride = alloc->imported.alias.stride;
853 KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased);
854 for (i = 0; i < alloc->imported.alias.nents; i++) {
855 if (alloc->imported.alias.aliased[i].alloc) {
856 err = kbase_mmu_insert_pages(kctx,
857 reg->start_pfn + (i * stride),
858 alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset,
859 alloc->imported.alias.aliased[i].length,
864 kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc);
866 err = kbase_mmu_insert_single_page(kctx,
867 reg->start_pfn + i * stride,
868 page_to_phys(kctx->aliasing_sink_page),
869 alloc->imported.alias.aliased[i].length,
870 (reg->flags & mask) | attr);
877 err = kbase_mmu_insert_pages(kctx, reg->start_pfn,
878 kbase_get_gpu_phy_pages(reg),
879 kbase_reg_current_backed_size(reg),
883 kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc);
889 if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
892 stride = reg->gpu_alloc->imported.alias.stride;
893 KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
895 if (reg->gpu_alloc->imported.alias.aliased[i].alloc) {
896 kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length);
897 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
901 kbase_remove_va_region(kctx, reg);
906 KBASE_EXPORT_TEST_API(kbase_gpu_mmap);
908 int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
912 if (reg->start_pfn == 0)
915 if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
918 err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages);
919 KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
920 for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++)
921 if (reg->gpu_alloc->imported.alias.aliased[i].alloc)
922 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
924 err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg));
925 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc);
931 err = kbase_remove_va_region(kctx, reg);
935 static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping_of_region(const struct kbase_va_region *reg, unsigned long uaddr, size_t size)
937 struct kbase_cpu_mapping *map;
938 struct list_head *pos;
940 KBASE_DEBUG_ASSERT(NULL != reg);
941 KBASE_DEBUG_ASSERT(reg->cpu_alloc);
943 if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */
946 list_for_each(pos, ®->cpu_alloc->mappings) {
947 map = list_entry(pos, struct kbase_cpu_mapping, mappings_list);
948 if (map->vm_start <= uaddr && map->vm_end >= uaddr + size)
955 KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_of_region);
957 int kbasep_find_enclosing_cpu_mapping_offset(
958 struct kbase_context *kctx, u64 gpu_addr,
959 unsigned long uaddr, size_t size, u64 * offset)
961 struct kbase_cpu_mapping *map = NULL;
962 const struct kbase_va_region *reg;
965 KBASE_DEBUG_ASSERT(kctx != NULL);
967 kbase_gpu_vm_lock(kctx);
969 reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
970 if (reg && !(reg->flags & KBASE_REG_FREE)) {
971 map = kbasep_find_enclosing_cpu_mapping_of_region(reg, uaddr,
974 *offset = (uaddr - PTR_TO_U64(map->vm_start)) +
975 (map->page_off << PAGE_SHIFT);
980 kbase_gpu_vm_unlock(kctx);
985 KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset);
987 void kbase_sync_single(struct kbase_context *kctx,
988 phys_addr_t cpu_pa, phys_addr_t gpu_pa,
989 off_t offset, size_t size, enum kbase_sync_type sync_fn)
991 struct page *cpu_page;
993 cpu_page = pfn_to_page(PFN_DOWN(cpu_pa));
995 if (likely(cpu_pa == gpu_pa)) {
999 BUG_ON(offset + size > PAGE_SIZE);
1001 dma_addr = kbase_dma_addr(cpu_page) + offset;
1002 if (sync_fn == KBASE_SYNC_TO_CPU)
1003 dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr,
1004 size, DMA_BIDIRECTIONAL);
1005 else if (sync_fn == KBASE_SYNC_TO_DEVICE)
1006 dma_sync_single_for_device(kctx->kbdev->dev, dma_addr,
1007 size, DMA_BIDIRECTIONAL);
1011 struct page *gpu_page;
1013 if (WARN(!gpu_pa, "No GPU PA found for infinite cache op"))
1016 gpu_page = pfn_to_page(PFN_DOWN(gpu_pa));
1018 if (sync_fn == KBASE_SYNC_TO_DEVICE) {
1019 src = ((unsigned char *)kmap(cpu_page)) + offset;
1020 dst = ((unsigned char *)kmap(gpu_page)) + offset;
1021 } else if (sync_fn == KBASE_SYNC_TO_CPU) {
1022 dma_sync_single_for_cpu(kctx->kbdev->dev,
1023 kbase_dma_addr(gpu_page) + offset,
1024 size, DMA_BIDIRECTIONAL);
1025 src = ((unsigned char *)kmap(gpu_page)) + offset;
1026 dst = ((unsigned char *)kmap(cpu_page)) + offset;
1028 memcpy(dst, src, size);
1031 if (sync_fn == KBASE_SYNC_TO_DEVICE)
1032 dma_sync_single_for_device(kctx->kbdev->dev,
1033 kbase_dma_addr(gpu_page) + offset,
1034 size, DMA_BIDIRECTIONAL);
1038 static int kbase_do_syncset(struct kbase_context *kctx,
1039 struct base_syncset *set, enum kbase_sync_type sync_fn)
1042 struct basep_syncset *sset = &set->basep_sset;
1043 struct kbase_va_region *reg;
1044 struct kbase_cpu_mapping *map;
1045 unsigned long start;
1047 phys_addr_t *cpu_pa;
1048 phys_addr_t *gpu_pa;
1049 u64 page_off, page_count;
1053 kbase_os_mem_map_lock(kctx);
1054 kbase_gpu_vm_lock(kctx);
1056 /* find the region where the virtual address is contained */
1057 reg = kbase_region_tracker_find_region_enclosing_address(kctx,
1058 sset->mem_handle.basep.handle);
1060 dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX",
1061 sset->mem_handle.basep.handle);
1066 if (!(reg->flags & KBASE_REG_CPU_CACHED))
1069 start = (uintptr_t)sset->user_addr;
1070 size = (size_t)sset->size;
1072 map = kbasep_find_enclosing_cpu_mapping_of_region(reg, start, size);
1074 dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX",
1075 start, sset->mem_handle.basep.handle);
1080 offset = start & (PAGE_SIZE - 1);
1081 page_off = map->page_off + ((start - map->vm_start) >> PAGE_SHIFT);
1082 page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
1083 cpu_pa = kbase_get_cpu_phy_pages(reg);
1084 gpu_pa = kbase_get_gpu_phy_pages(reg);
1086 /* Sync first page */
1087 if (cpu_pa[page_off]) {
1088 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
1090 kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off],
1091 offset, sz, sync_fn);
1094 /* Sync middle pages (if any) */
1095 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
1096 /* we grow upwards, so bail on first non-present page */
1097 if (!cpu_pa[page_off + i])
1100 kbase_sync_single(kctx, cpu_pa[page_off + i],
1101 gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn);
1104 /* Sync last page (if any) */
1105 if (page_count > 1 && cpu_pa[page_off + page_count - 1]) {
1106 size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1;
1108 kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1],
1109 gpu_pa[page_off + page_count - 1], 0, sz,
1114 kbase_gpu_vm_unlock(kctx);
1115 kbase_os_mem_map_unlock(kctx);
1119 int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset)
1122 struct basep_syncset *sset;
1124 KBASE_DEBUG_ASSERT(NULL != kctx);
1125 KBASE_DEBUG_ASSERT(NULL != syncset);
1127 sset = &syncset->basep_sset;
1129 switch (sset->type) {
1130 case BASE_SYNCSET_OP_MSYNC:
1131 err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_DEVICE);
1134 case BASE_SYNCSET_OP_CSYNC:
1135 err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_CPU);
1139 dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type);
1146 KBASE_EXPORT_TEST_API(kbase_sync_now);
1148 /* vm lock must be held */
1149 int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg)
1153 KBASE_DEBUG_ASSERT(NULL != kctx);
1154 KBASE_DEBUG_ASSERT(NULL != reg);
1155 lockdep_assert_held(&kctx->reg_lock);
1158 * Unlink the physical allocation before unmaking it evictable so
1159 * that the allocation isn't grown back to its last backed size
1160 * as we're going to unmap it anyway.
1162 reg->cpu_alloc->reg = NULL;
1163 if (reg->cpu_alloc != reg->gpu_alloc)
1164 reg->gpu_alloc->reg = NULL;
1167 * If a region has been made evictable then we must unmake it
1168 * before trying to free it.
1169 * If the memory hasn't been reclaimed it will be unmapped and freed
1170 * below, if it has been reclaimed then the operations below are no-ops.
1172 if (reg->flags & KBASE_REG_DONT_NEED) {
1173 KBASE_DEBUG_ASSERT(reg->cpu_alloc->type ==
1174 KBASE_MEM_TYPE_NATIVE);
1175 kbase_mem_evictable_unmake(reg->gpu_alloc);
1178 err = kbase_gpu_munmap(kctx, reg);
1180 dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n");
1183 #ifndef CONFIG_MALI_NO_MALI
1184 if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) {
1185 /* Wait for GPU to flush write buffer before freeing physical pages */
1186 kbase_wait_write_flush(kctx);
1189 /* This will also free the physical pages */
1190 kbase_free_alloced_region(reg);
1196 KBASE_EXPORT_TEST_API(kbase_mem_free_region);
1199 * @brief Free the region from the GPU and unregister it.
1201 * This function implements the free operation on a memory segment.
1202 * It will loudly fail if called with outstanding mappings.
1204 int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr)
1207 struct kbase_va_region *reg;
1209 KBASE_DEBUG_ASSERT(kctx != NULL);
1211 if (0 == gpu_addr) {
1212 dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n");
1215 kbase_gpu_vm_lock(kctx);
1217 if (gpu_addr >= BASE_MEM_COOKIE_BASE &&
1218 gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) {
1219 int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE);
1221 reg = kctx->pending_regions[cookie];
1227 /* ask to unlink the cookie as we'll free it */
1229 kctx->pending_regions[cookie] = NULL;
1230 kctx->cookies |= (1UL << cookie);
1232 kbase_free_alloced_region(reg);
1235 /* Validate the region */
1236 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1237 if (!reg || (reg->flags & KBASE_REG_FREE)) {
1238 dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX",
1244 if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) {
1245 /* SAME_VA must be freed through munmap */
1246 dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__,
1251 err = kbase_mem_free_region(kctx, reg);
1255 kbase_gpu_vm_unlock(kctx);
1259 KBASE_EXPORT_TEST_API(kbase_mem_free);
1261 void kbase_update_region_flags(struct kbase_context *kctx,
1262 struct kbase_va_region *reg, unsigned long flags)
1264 KBASE_DEBUG_ASSERT(NULL != reg);
1265 KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0);
1267 reg->flags |= kbase_cache_enabled(flags, reg->nr_pages);
1268 /* all memory is now growable */
1269 reg->flags |= KBASE_REG_GROWABLE;
1271 if (flags & BASE_MEM_GROW_ON_GPF)
1272 reg->flags |= KBASE_REG_PF_GROW;
1274 if (flags & BASE_MEM_PROT_CPU_WR)
1275 reg->flags |= KBASE_REG_CPU_WR;
1277 if (flags & BASE_MEM_PROT_CPU_RD)
1278 reg->flags |= KBASE_REG_CPU_RD;
1280 if (flags & BASE_MEM_PROT_GPU_WR)
1281 reg->flags |= KBASE_REG_GPU_WR;
1283 if (flags & BASE_MEM_PROT_GPU_RD)
1284 reg->flags |= KBASE_REG_GPU_RD;
1286 if (0 == (flags & BASE_MEM_PROT_GPU_EX))
1287 reg->flags |= KBASE_REG_GPU_NX;
1289 if (flags & BASE_MEM_COHERENT_SYSTEM ||
1290 flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED)
1291 reg->flags |= KBASE_REG_SHARE_BOTH;
1292 else if (flags & BASE_MEM_COHERENT_LOCAL)
1293 reg->flags |= KBASE_REG_SHARE_IN;
1295 /* Set up default MEMATTR usage */
1296 if (kctx->kbdev->system_coherency == COHERENCY_ACE &&
1297 (reg->flags & KBASE_REG_SHARE_BOTH)) {
1299 KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE);
1302 KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT);
1305 KBASE_EXPORT_TEST_API(kbase_update_region_flags);
1307 int kbase_alloc_phy_pages_helper(
1308 struct kbase_mem_phy_alloc *alloc,
1309 size_t nr_pages_requested)
1311 int new_page_count __maybe_unused;
1313 KBASE_DEBUG_ASSERT(alloc);
1314 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE);
1315 KBASE_DEBUG_ASSERT(alloc->imported.kctx);
1317 if (nr_pages_requested == 0)
1318 goto done; /*nothing to do*/
1320 new_page_count = kbase_atomic_add_pages(
1321 nr_pages_requested, &alloc->imported.kctx->used_pages);
1322 kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages);
1324 /* Increase mm counters before we allocate pages so that this
1325 * allocation is visible to the OOM killer */
1326 kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested);
1328 if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool,
1329 nr_pages_requested, alloc->pages + alloc->nents) != 0)
1332 kbase_tlstream_aux_pagesalloc(
1333 (u32)alloc->imported.kctx->id,
1334 (u64)new_page_count);
1336 alloc->nents += nr_pages_requested;
1341 kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_requested);
1342 kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->used_pages);
1343 kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages);
1348 int kbase_free_phy_pages_helper(
1349 struct kbase_mem_phy_alloc *alloc,
1350 size_t nr_pages_to_free)
1352 struct kbase_context *kctx = alloc->imported.kctx;
1354 bool reclaimed = (alloc->evicted != 0);
1355 phys_addr_t *start_free;
1356 int new_page_count __maybe_unused;
1358 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE);
1359 KBASE_DEBUG_ASSERT(alloc->imported.kctx);
1360 KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free);
1362 /* early out if nothing to do */
1363 if (0 == nr_pages_to_free)
1366 start_free = alloc->pages + alloc->nents - nr_pages_to_free;
1368 syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
1370 kbase_mem_pool_free_pages(&kctx->mem_pool,
1376 alloc->nents -= nr_pages_to_free;
1379 * If the allocation was not evicted (i.e. evicted == 0) then
1380 * the page accounting needs to be done.
1383 kbase_process_page_usage_dec(kctx, nr_pages_to_free);
1384 new_page_count = kbase_atomic_sub_pages(nr_pages_to_free,
1386 kbase_atomic_sub_pages(nr_pages_to_free,
1387 &kctx->kbdev->memdev.used_pages);
1389 kbase_tlstream_aux_pagesalloc(
1391 (u64)new_page_count);
1397 void kbase_mem_kref_free(struct kref *kref)
1399 struct kbase_mem_phy_alloc *alloc;
1401 alloc = container_of(kref, struct kbase_mem_phy_alloc, kref);
1403 switch (alloc->type) {
1404 case KBASE_MEM_TYPE_NATIVE: {
1405 WARN_ON(!alloc->imported.kctx);
1407 * The physical allocation must have been removed from the
1408 * eviction list before trying to free it.
1410 WARN_ON(!list_empty(&alloc->evict_node));
1411 kbase_free_phy_pages_helper(alloc, alloc->nents);
1414 case KBASE_MEM_TYPE_ALIAS: {
1415 /* just call put on the underlying phy allocs */
1417 struct kbase_aliased *aliased;
1419 aliased = alloc->imported.alias.aliased;
1421 for (i = 0; i < alloc->imported.alias.nents; i++)
1422 if (aliased[i].alloc)
1423 kbase_mem_phy_alloc_put(aliased[i].alloc);
1428 case KBASE_MEM_TYPE_RAW:
1429 /* raw pages, external cleanup */
1432 case KBASE_MEM_TYPE_IMPORTED_UMP:
1433 ump_dd_release(alloc->imported.ump_handle);
1436 #ifdef CONFIG_DMA_SHARED_BUFFER
1437 case KBASE_MEM_TYPE_IMPORTED_UMM:
1438 dma_buf_detach(alloc->imported.umm.dma_buf,
1439 alloc->imported.umm.dma_attachment);
1440 dma_buf_put(alloc->imported.umm.dma_buf);
1443 case KBASE_MEM_TYPE_IMPORTED_USER_BUF:
1444 kfree(alloc->imported.user_buf.pages);
1446 case KBASE_MEM_TYPE_TB:{
1449 tb = alloc->imported.kctx->jctx.tb;
1450 kbase_device_trace_buffer_uninstall(alloc->imported.kctx);
1455 WARN(1, "Unexecpted free of type %d\n", alloc->type);
1459 /* Free based on allocation type */
1460 if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE)
1466 KBASE_EXPORT_TEST_API(kbase_mem_kref_free);
1468 int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size)
1470 KBASE_DEBUG_ASSERT(NULL != reg);
1471 KBASE_DEBUG_ASSERT(vsize > 0);
1473 /* validate user provided arguments */
1474 if (size > vsize || vsize > reg->nr_pages)
1477 /* Prevent vsize*sizeof from wrapping around.
1478 * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail.
1480 if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages)))
1483 KBASE_DEBUG_ASSERT(0 != vsize);
1485 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0)
1488 reg->cpu_alloc->reg = reg;
1489 if (reg->cpu_alloc != reg->gpu_alloc) {
1490 if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0)
1492 reg->gpu_alloc->reg = reg;
1498 kbase_free_phy_pages_helper(reg->cpu_alloc, size);
1503 KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages);
1505 bool kbase_check_alloc_flags(unsigned long flags)
1507 /* Only known input flags should be set. */
1508 if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
1511 /* At least one flag should be set */
1515 /* Either the GPU or CPU must be reading from the allocated memory */
1516 if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0)
1519 /* Either the GPU or CPU must be writing to the allocated memory */
1520 if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0)
1523 /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */
1524 if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF)))
1527 /* GPU should have at least read or write access otherwise there is no
1528 reason for allocating. */
1529 if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
1535 bool kbase_check_import_flags(unsigned long flags)
1537 /* Only known input flags should be set. */
1538 if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
1541 /* At least one flag should be set */
1545 /* Imported memory cannot be GPU executable */
1546 if (flags & BASE_MEM_PROT_GPU_EX)
1549 /* Imported memory cannot grow on page fault */
1550 if (flags & BASE_MEM_GROW_ON_GPF)
1553 /* GPU should have at least read or write access otherwise there is no
1554 reason for importing. */
1555 if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
1558 /* Secure memory cannot be read by the CPU */
1559 if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD))
1566 * @brief Acquire the per-context region list lock
1568 void kbase_gpu_vm_lock(struct kbase_context *kctx)
1570 KBASE_DEBUG_ASSERT(kctx != NULL);
1571 mutex_lock(&kctx->reg_lock);
1574 KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock);
1577 * @brief Release the per-context region list lock
1579 void kbase_gpu_vm_unlock(struct kbase_context *kctx)
1581 KBASE_DEBUG_ASSERT(kctx != NULL);
1582 mutex_unlock(&kctx->reg_lock);
1585 KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock);
1587 struct kbase_jit_debugfs_data {
1588 int (*func)(struct kbase_jit_debugfs_data *);
1590 struct kbase_context *kctx;
1597 static int kbase_jit_debugfs_common_open(struct inode *inode,
1598 struct file *file, int (*func)(struct kbase_jit_debugfs_data *))
1600 struct kbase_jit_debugfs_data *data;
1602 data = kzalloc(sizeof(*data), GFP_KERNEL);
1607 mutex_init(&data->lock);
1608 data->kctx = (struct kbase_context *) inode->i_private;
1610 file->private_data = data;
1612 return nonseekable_open(inode, file);
1615 static ssize_t kbase_jit_debugfs_common_read(struct file *file,
1616 char __user *buf, size_t len, loff_t *ppos)
1618 struct kbase_jit_debugfs_data *data;
1622 data = (struct kbase_jit_debugfs_data *) file->private_data;
1623 mutex_lock(&data->lock);
1626 size = strnlen(data->buffer, sizeof(data->buffer));
1633 if (data->func(data)) {
1638 size = scnprintf(data->buffer, sizeof(data->buffer),
1639 "%llu,%llu,%llu", data->active_value,
1640 data->pool_value, data->destroy_value);
1643 ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size);
1646 mutex_unlock(&data->lock);
1650 static int kbase_jit_debugfs_common_release(struct inode *inode,
1653 kfree(file->private_data);
1657 #define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \
1658 static int __fops ## _open(struct inode *inode, struct file *file) \
1660 return kbase_jit_debugfs_common_open(inode, file, __func); \
1662 static const struct file_operations __fops = { \
1663 .owner = THIS_MODULE, \
1664 .open = __fops ## _open, \
1665 .release = kbase_jit_debugfs_common_release, \
1666 .read = kbase_jit_debugfs_common_read, \
1668 .llseek = generic_file_llseek, \
1671 static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data)
1673 struct kbase_context *kctx = data->kctx;
1674 struct list_head *tmp;
1676 mutex_lock(&kctx->jit_lock);
1677 list_for_each(tmp, &kctx->jit_active_head) {
1678 data->active_value++;
1681 list_for_each(tmp, &kctx->jit_pool_head) {
1685 list_for_each(tmp, &kctx->jit_destroy_head) {
1686 data->destroy_value++;
1688 mutex_unlock(&kctx->jit_lock);
1692 KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops,
1693 kbase_jit_debugfs_count_get);
1695 static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data)
1697 struct kbase_context *kctx = data->kctx;
1698 struct kbase_va_region *reg;
1700 mutex_lock(&kctx->jit_lock);
1701 list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
1702 data->active_value += reg->nr_pages;
1705 list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) {
1706 data->pool_value += reg->nr_pages;
1709 list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) {
1710 data->destroy_value += reg->nr_pages;
1712 mutex_unlock(&kctx->jit_lock);
1716 KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops,
1717 kbase_jit_debugfs_vm_get);
1719 static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data)
1721 struct kbase_context *kctx = data->kctx;
1722 struct kbase_va_region *reg;
1724 mutex_lock(&kctx->jit_lock);
1725 list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
1726 data->active_value += reg->gpu_alloc->nents;
1729 list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) {
1730 data->pool_value += reg->gpu_alloc->nents;
1733 list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) {
1734 data->destroy_value += reg->gpu_alloc->nents;
1736 mutex_unlock(&kctx->jit_lock);
1740 KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops,
1741 kbase_jit_debugfs_phys_get);
1743 void kbase_jit_debugfs_add(struct kbase_context *kctx)
1745 /* Debugfs entry for getting the number of JIT allocations. */
1746 debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry,
1747 kctx, &kbase_jit_debugfs_count_fops);
1750 * Debugfs entry for getting the total number of virtual pages
1751 * used by JIT allocations.
1753 debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry,
1754 kctx, &kbase_jit_debugfs_vm_fops);
1757 * Debugfs entry for getting the number of physical pages used
1758 * by JIT allocations.
1760 debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry,
1761 kctx, &kbase_jit_debugfs_phys_fops);
1765 * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations
1768 * This function does the work of freeing JIT allocations whose physical
1769 * backing has been released.
1771 static void kbase_jit_destroy_worker(struct work_struct *work)
1773 struct kbase_context *kctx;
1774 struct kbase_va_region *reg;
1776 kctx = container_of(work, struct kbase_context, jit_work);
1778 mutex_lock(&kctx->jit_lock);
1779 if (list_empty(&kctx->jit_destroy_head))
1782 reg = list_first_entry(&kctx->jit_destroy_head,
1783 struct kbase_va_region, jit_node);
1786 list_del(®->jit_node);
1787 mutex_unlock(&kctx->jit_lock);
1789 kbase_gpu_vm_lock(kctx);
1790 kbase_mem_free_region(kctx, reg);
1791 kbase_gpu_vm_unlock(kctx);
1793 mutex_unlock(&kctx->jit_lock);
1797 int kbase_jit_init(struct kbase_context *kctx)
1799 INIT_LIST_HEAD(&kctx->jit_active_head);
1800 INIT_LIST_HEAD(&kctx->jit_pool_head);
1801 INIT_LIST_HEAD(&kctx->jit_destroy_head);
1802 mutex_init(&kctx->jit_lock);
1803 INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker);
1808 struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
1809 struct base_jit_alloc_info *info)
1811 struct kbase_va_region *reg = NULL;
1812 struct kbase_va_region *walker;
1813 struct kbase_va_region *temp;
1814 size_t current_diff = SIZE_MAX;
1818 mutex_lock(&kctx->jit_lock);
1820 * Scan the pool for an existing allocation which meets our
1821 * requirements and remove it.
1823 list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) {
1825 if (walker->nr_pages >= info->va_pages) {
1826 size_t min_size, max_size, diff;
1829 * The JIT allocations VA requirements have been
1830 * meet, it's suitable but other allocations
1831 * might be a better fit.
1833 min_size = min_t(size_t, walker->gpu_alloc->nents,
1834 info->commit_pages);
1835 max_size = max_t(size_t, walker->gpu_alloc->nents,
1836 info->commit_pages);
1837 diff = max_size - min_size;
1839 if (current_diff > diff) {
1840 current_diff = diff;
1844 /* The allocation is an exact match, stop looking */
1845 if (current_diff == 0)
1852 * Remove the found region from the pool and add it to the
1855 list_del_init(®->jit_node);
1856 list_add(®->jit_node, &kctx->jit_active_head);
1858 /* Release the jit lock before modifying the allocation */
1859 mutex_unlock(&kctx->jit_lock);
1861 kbase_gpu_vm_lock(kctx);
1863 /* Make the physical backing no longer reclaimable */
1864 if (!kbase_mem_evictable_unmake(reg->gpu_alloc))
1867 /* Grow the backing if required */
1868 if (reg->gpu_alloc->nents < info->commit_pages) {
1870 size_t old_size = reg->gpu_alloc->nents;
1872 /* Allocate some more pages */
1873 delta = info->commit_pages - reg->gpu_alloc->nents;
1874 if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta)
1878 if (reg->cpu_alloc != reg->gpu_alloc) {
1879 if (kbase_alloc_phy_pages_helper(
1880 reg->cpu_alloc, delta) != 0) {
1881 kbase_free_phy_pages_helper(
1882 reg->gpu_alloc, delta);
1887 ret = kbase_mem_grow_gpu_mapping(kctx, reg,
1888 info->commit_pages, old_size);
1890 * The grow failed so put the allocation back in the
1891 * pool and return failure.
1896 kbase_gpu_vm_unlock(kctx);
1898 /* No suitable JIT allocation was found so create a new one */
1899 u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD |
1900 BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF;
1904 mutex_unlock(&kctx->jit_lock);
1906 reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages,
1907 info->extent, &flags, &gpu_addr, &alignment);
1911 mutex_lock(&kctx->jit_lock);
1912 list_add(®->jit_node, &kctx->jit_active_head);
1913 mutex_unlock(&kctx->jit_lock);
1920 * An update to an allocation from the pool failed, chances
1921 * are slim a new allocation would fair any better so return
1922 * the allocation to the pool and return the function with failure.
1924 kbase_gpu_vm_unlock(kctx);
1925 mutex_lock(&kctx->jit_lock);
1926 list_del_init(®->jit_node);
1927 list_add(®->jit_node, &kctx->jit_pool_head);
1928 mutex_unlock(&kctx->jit_lock);
1933 void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg)
1935 /* The physical backing of memory in the pool is always reclaimable */
1936 down_read(&kctx->process_mm->mmap_sem);
1937 kbase_gpu_vm_lock(kctx);
1938 kbase_mem_evictable_make(reg->gpu_alloc);
1939 kbase_gpu_vm_unlock(kctx);
1940 up_read(&kctx->process_mm->mmap_sem);
1942 mutex_lock(&kctx->jit_lock);
1943 list_del_init(®->jit_node);
1944 list_add(®->jit_node, &kctx->jit_pool_head);
1945 mutex_unlock(&kctx->jit_lock);
1948 void kbase_jit_backing_lost(struct kbase_va_region *reg)
1950 struct kbase_context *kctx = reg->kctx;
1953 * JIT allocations will always be on a list, if the region
1954 * is not on a list then it's not a JIT allocation.
1956 if (list_empty(®->jit_node))
1960 * Freeing the allocation requires locks we might not be able
1961 * to take now, so move the allocation to the free list and kick
1962 * the worker which will do the freeing.
1964 mutex_lock(&kctx->jit_lock);
1965 list_del_init(®->jit_node);
1966 list_add(®->jit_node, &kctx->jit_destroy_head);
1967 mutex_unlock(&kctx->jit_lock);
1969 schedule_work(&kctx->jit_work);
1972 bool kbase_jit_evict(struct kbase_context *kctx)
1974 struct kbase_va_region *reg = NULL;
1976 lockdep_assert_held(&kctx->reg_lock);
1978 /* Free the oldest allocation from the pool */
1979 mutex_lock(&kctx->jit_lock);
1980 if (!list_empty(&kctx->jit_pool_head)) {
1981 reg = list_entry(kctx->jit_pool_head.prev,
1982 struct kbase_va_region, jit_node);
1983 list_del(®->jit_node);
1985 mutex_unlock(&kctx->jit_lock);
1988 kbase_mem_free_region(kctx, reg);
1990 return (reg != NULL);
1993 void kbase_jit_term(struct kbase_context *kctx)
1995 struct kbase_va_region *walker;
1997 /* Free all allocations for this context */
2000 * Flush the freeing of allocations whose backing has been freed
2001 * (i.e. everything in jit_destroy_head).
2003 cancel_work_sync(&kctx->jit_work);
2005 kbase_gpu_vm_lock(kctx);
2006 /* Free all allocations from the pool */
2007 while (!list_empty(&kctx->jit_pool_head)) {
2008 walker = list_first_entry(&kctx->jit_pool_head,
2009 struct kbase_va_region, jit_node);
2010 list_del(&walker->jit_node);
2011 kbase_mem_free_region(kctx, walker);
2014 /* Free all allocations from active list */
2015 while (!list_empty(&kctx->jit_active_head)) {
2016 walker = list_first_entry(&kctx->jit_active_head,
2017 struct kbase_va_region, jit_node);
2018 list_del(&walker->jit_node);
2019 kbase_mem_free_region(kctx, walker);
2021 kbase_gpu_vm_unlock(kctx);
2024 static int kbase_jd_user_buf_map(struct kbase_context *kctx,
2025 struct kbase_va_region *reg)
2028 struct kbase_mem_phy_alloc *alloc;
2029 struct page **pages;
2033 unsigned long address;
2034 struct task_struct *owner;
2036 unsigned long offset;
2037 unsigned long local_size;
2039 alloc = reg->gpu_alloc;
2040 pa = kbase_get_gpu_phy_pages(reg);
2041 address = alloc->imported.user_buf.address;
2042 owner = alloc->imported.user_buf.owner;
2044 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
2046 pages = alloc->imported.user_buf.pages;
2048 pinned_pages = get_user_pages(owner, owner->mm,
2050 alloc->imported.user_buf.nr_pages,
2051 reg->flags & KBASE_REG_GPU_WR,
2054 if (pinned_pages <= 0)
2055 return pinned_pages;
2057 if (pinned_pages != alloc->imported.user_buf.nr_pages) {
2058 for (i = 0; i < pinned_pages; i++)
2063 dev = kctx->kbdev->dev;
2064 offset = address & ~PAGE_MASK;
2065 local_size = alloc->imported.user_buf.size;
2067 for (i = 0; i < pinned_pages; i++) {
2068 dma_addr_t dma_addr;
2071 min = MIN(PAGE_SIZE - offset, local_size);
2072 dma_addr = dma_map_page(dev, pages[i],
2075 if (dma_mapping_error(dev, dma_addr))
2078 alloc->imported.user_buf.dma_addrs[i] = dma_addr;
2079 pa[i] = page_to_phys(pages[i]);
2085 alloc->nents = pinned_pages;
2087 err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa,
2088 kbase_reg_current_backed_size(reg),
2097 dma_unmap_page(kctx->kbdev->dev,
2098 alloc->imported.user_buf.dma_addrs[i],
2099 PAGE_SIZE, DMA_BIDIRECTIONAL);
2107 static void kbase_jd_user_buf_unmap(struct kbase_context *kctx,
2108 struct kbase_mem_phy_alloc *alloc, bool writeable)
2111 struct page **pages;
2112 unsigned long size = alloc->imported.user_buf.size;
2114 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
2115 pages = alloc->imported.user_buf.pages;
2116 for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) {
2117 unsigned long local_size;
2118 dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i];
2120 local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK));
2121 dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size,
2124 set_page_dirty_lock(pages[i]);
2133 #ifdef CONFIG_DMA_SHARED_BUFFER
2134 static int kbase_jd_umm_map(struct kbase_context *kctx,
2135 struct kbase_va_region *reg)
2137 struct sg_table *sgt;
2138 struct scatterlist *s;
2143 struct kbase_mem_phy_alloc *alloc;
2145 alloc = reg->gpu_alloc;
2147 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM);
2148 KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt);
2149 sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment,
2152 if (IS_ERR_OR_NULL(sgt))
2155 /* save for later */
2156 alloc->imported.umm.sgt = sgt;
2158 pa = kbase_get_gpu_phy_pages(reg);
2159 KBASE_DEBUG_ASSERT(pa);
2161 for_each_sg(sgt->sgl, s, sgt->nents, i) {
2163 size_t pages = PFN_UP(sg_dma_len(s));
2165 WARN_ONCE(sg_dma_len(s) & (PAGE_SIZE-1),
2166 "sg_dma_len(s)=%u is not a multiple of PAGE_SIZE\n",
2169 WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1),
2170 "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n",
2171 (unsigned long long) sg_dma_address(s));
2173 for (j = 0; (j < pages) && (count < reg->nr_pages); j++,
2175 *pa++ = sg_dma_address(s) + (j << PAGE_SHIFT);
2176 WARN_ONCE(j < pages,
2177 "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n",
2178 alloc->imported.umm.dma_buf->size);
2181 if (WARN_ONCE(count < reg->nr_pages,
2182 "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n",
2183 alloc->imported.umm.dma_buf->size)) {
2188 /* Update nents as we now have pages to map */
2189 alloc->nents = count;
2191 err = kbase_mmu_insert_pages(kctx, reg->start_pfn,
2192 kbase_get_gpu_phy_pages(reg),
2193 kbase_reg_current_backed_size(reg),
2194 reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);
2198 dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment,
2199 alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
2200 alloc->imported.umm.sgt = NULL;
2206 static void kbase_jd_umm_unmap(struct kbase_context *kctx,
2207 struct kbase_mem_phy_alloc *alloc)
2209 KBASE_DEBUG_ASSERT(kctx);
2210 KBASE_DEBUG_ASSERT(alloc);
2211 KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment);
2212 KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt);
2213 dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment,
2214 alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
2215 alloc->imported.umm.sgt = NULL;
2218 #endif /* CONFIG_DMA_SHARED_BUFFER */
2220 #if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \
2221 || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS)
2222 static void add_kds_resource(struct kds_resource *kds_res,
2223 struct kds_resource **kds_resources, u32 *kds_res_count,
2224 unsigned long *kds_access_bitmap, bool exclusive)
2228 for (i = 0; i < *kds_res_count; i++) {
2229 /* Duplicate resource, ignore */
2230 if (kds_resources[i] == kds_res)
2234 kds_resources[*kds_res_count] = kds_res;
2236 set_bit(*kds_res_count, kds_access_bitmap);
2241 struct kbase_mem_phy_alloc *kbase_map_external_resource(
2242 struct kbase_context *kctx, struct kbase_va_region *reg,
2243 struct mm_struct *locked_mm
2245 , u32 *kds_res_count, struct kds_resource **kds_resources,
2246 unsigned long *kds_access_bitmap, bool exclusive
2252 /* decide what needs to happen for this resource */
2253 switch (reg->gpu_alloc->type) {
2254 case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
2255 if (reg->gpu_alloc->imported.user_buf.owner->mm != locked_mm)
2258 reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++;
2259 if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) {
2260 err = kbase_jd_user_buf_map(kctx, reg);
2262 reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--;
2268 case BASE_MEM_IMPORT_TYPE_UMP: {
2269 #if defined(CONFIG_KDS) && defined(CONFIG_UMP)
2270 if (kds_res_count) {
2271 struct kds_resource *kds_res;
2273 kds_res = ump_dd_kds_resource_get(
2274 reg->gpu_alloc->imported.ump_handle);
2276 add_kds_resource(kds_res, kds_resources,
2278 kds_access_bitmap, exclusive);
2280 #endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */
2283 #ifdef CONFIG_DMA_SHARED_BUFFER
2284 case BASE_MEM_IMPORT_TYPE_UMM: {
2285 #ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS
2286 if (kds_res_count) {
2287 struct kds_resource *kds_res;
2289 kds_res = get_dma_buf_kds_resource(
2290 reg->gpu_alloc->imported.umm.dma_buf);
2292 add_kds_resource(kds_res, kds_resources,
2294 kds_access_bitmap, exclusive);
2297 reg->gpu_alloc->imported.umm.current_mapping_usage_count++;
2298 if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) {
2299 err = kbase_jd_umm_map(kctx, reg);
2301 reg->gpu_alloc->imported.umm.current_mapping_usage_count--;
2312 return kbase_mem_phy_alloc_get(reg->gpu_alloc);
2317 void kbase_unmap_external_resource(struct kbase_context *kctx,
2318 struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc)
2320 switch (alloc->type) {
2321 #ifdef CONFIG_DMA_SHARED_BUFFER
2322 case KBASE_MEM_TYPE_IMPORTED_UMM: {
2323 alloc->imported.umm.current_mapping_usage_count--;
2325 if (0 == alloc->imported.umm.current_mapping_usage_count) {
2326 if (reg && reg->gpu_alloc == alloc)
2327 kbase_mmu_teardown_pages(
2330 kbase_reg_current_backed_size(reg));
2332 kbase_jd_umm_unmap(kctx, alloc);
2336 #endif /* CONFIG_DMA_SHARED_BUFFER */
2337 case KBASE_MEM_TYPE_IMPORTED_USER_BUF: {
2338 alloc->imported.user_buf.current_mapping_usage_count--;
2340 if (0 == alloc->imported.user_buf.current_mapping_usage_count) {
2341 bool writeable = true;
2343 if (reg && reg->gpu_alloc == alloc)
2344 kbase_mmu_teardown_pages(
2347 kbase_reg_current_backed_size(reg));
2349 if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0))
2352 kbase_jd_user_buf_unmap(kctx, alloc, writeable);
2359 kbase_mem_phy_alloc_put(alloc);
2362 struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(
2363 struct kbase_context *kctx, u64 gpu_addr)
2365 struct kbase_ctx_ext_res_meta *meta = NULL;
2366 struct kbase_ctx_ext_res_meta *walker;
2368 lockdep_assert_held(&kctx->reg_lock);
2371 * Walk the per context externel resource metadata list for the
2372 * metadata which matches the region which is being acquired.
2374 list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) {
2375 if (walker->gpu_addr == gpu_addr) {
2381 /* No metadata exists so create one. */
2383 struct kbase_va_region *reg;
2385 /* Find the region */
2386 reg = kbase_region_tracker_find_region_enclosing_address(
2388 if (NULL == reg || (reg->flags & KBASE_REG_FREE))
2391 /* Allocate the metadata object */
2392 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
2397 * Fill in the metadata object and acquire a reference
2398 * for the physical resource.
2400 meta->alloc = kbase_map_external_resource(kctx, reg, NULL
2410 meta->gpu_addr = reg->start_pfn << PAGE_SHIFT;
2413 list_add(&meta->ext_res_node, &kctx->ext_res_meta_head);
2415 if (meta->refcount == UINT_MAX)
2429 bool kbase_sticky_resource_release(struct kbase_context *kctx,
2430 struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr, bool force)
2432 struct kbase_ctx_ext_res_meta *walker;
2434 lockdep_assert_held(&kctx->reg_lock);
2436 /* Search of the metadata if one isn't provided. */
2439 * Walk the per context externel resource metadata list for the
2440 * metadata which matches the region which is being released.
2442 list_for_each_entry(walker, &kctx->ext_res_meta_head,
2444 if (walker->gpu_addr == gpu_addr) {
2451 /* No metadata so just return. */
2456 if ((meta->refcount == 0) || force) {
2458 * Last reference to the metadata, drop the physical memory
2459 * reference and free the metadata.
2461 struct kbase_va_region *reg;
2463 reg = kbase_region_tracker_find_region_enclosing_address(
2467 kbase_unmap_external_resource(kctx, reg, meta->alloc);
2468 list_del(&meta->ext_res_node);
2475 int kbase_sticky_resource_init(struct kbase_context *kctx)
2477 INIT_LIST_HEAD(&kctx->ext_res_meta_head);
2482 void kbase_sticky_resource_term(struct kbase_context *kctx)
2484 struct kbase_ctx_ext_res_meta *walker;
2486 lockdep_assert_held(&kctx->reg_lock);
2489 * Free any sticky resources which haven't been unmapped.
2492 * We don't care about refcounts at this point as no future
2493 * references to the meta data will be made.
2494 * Region termination would find these if we didn't free them
2495 * here, but it's more efficient if we do the clean up here.
2497 while (!list_empty(&kctx->ext_res_meta_head)) {
2498 walker = list_first_entry(&kctx->ext_res_meta_head,
2499 struct kbase_ctx_ext_res_meta, ext_res_node);
2501 kbase_sticky_resource_release(kctx, walker, 0, true);