3 * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_mem.c
22 * Base kernel memory APIs
24 #ifdef CONFIG_DMA_SHARED_BUFFER
25 #include <linux/dma-buf.h>
26 #endif /* CONFIG_DMA_SHARED_BUFFER */
28 #include <linux/kernel.h>
29 #include <linux/bug.h>
30 #include <linux/compat.h>
32 #include <mali_kbase_config.h>
33 #include <mali_kbase.h>
34 #include <mali_midg_regmap.h>
35 #include <mali_kbase_cache_policy.h>
36 #include <mali_kbase_hw.h>
37 #include <mali_kbase_gator.h>
38 #include <mali_kbase_hwaccess_time.h>
40 #if defined(CONFIG_MALI_MIPE_ENABLED)
41 #include <mali_kbase_tlstream.h>
45 * @brief Check the zone compatibility of two regions.
47 static int kbase_region_tracker_match_zone(struct kbase_va_region *reg1,
48 struct kbase_va_region *reg2)
50 return ((reg1->flags & KBASE_REG_ZONE_MASK) ==
51 (reg2->flags & KBASE_REG_ZONE_MASK));
54 KBASE_EXPORT_TEST_API(kbase_region_tracker_match_zone);
56 /* This function inserts a region into the tree. */
57 static void kbase_region_tracker_insert(struct kbase_context *kctx, struct kbase_va_region *new_reg)
59 u64 start_pfn = new_reg->start_pfn;
60 struct rb_node **link = &(kctx->reg_rbtree.rb_node);
61 struct rb_node *parent = NULL;
63 /* Find the right place in the tree using tree search */
65 struct kbase_va_region *old_reg;
68 old_reg = rb_entry(parent, struct kbase_va_region, rblink);
70 /* RBTree requires no duplicate entries. */
71 KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn);
73 if (old_reg->start_pfn > start_pfn)
74 link = &(*link)->rb_left;
76 link = &(*link)->rb_right;
79 /* Put the new node there, and rebalance tree */
80 rb_link_node(&(new_reg->rblink), parent, link);
81 rb_insert_color(&(new_reg->rblink), &(kctx->reg_rbtree));
84 /* Find allocated region enclosing free range. */
85 static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free(
86 struct kbase_context *kctx, u64 start_pfn, size_t nr_pages)
88 struct rb_node *rbnode;
89 struct kbase_va_region *reg;
91 u64 end_pfn = start_pfn + nr_pages;
93 rbnode = kctx->reg_rbtree.rb_node;
95 u64 tmp_start_pfn, tmp_end_pfn;
97 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
98 tmp_start_pfn = reg->start_pfn;
99 tmp_end_pfn = reg->start_pfn + reg->nr_pages;
101 /* If start is lower than this, go left. */
102 if (start_pfn < tmp_start_pfn)
103 rbnode = rbnode->rb_left;
104 /* If end is higher than this, then go right. */
105 else if (end_pfn > tmp_end_pfn)
106 rbnode = rbnode->rb_right;
114 /* Find region enclosing given address. */
115 struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr)
117 struct rb_node *rbnode;
118 struct kbase_va_region *reg;
119 u64 gpu_pfn = gpu_addr >> PAGE_SHIFT;
121 KBASE_DEBUG_ASSERT(NULL != kctx);
123 lockdep_assert_held(&kctx->reg_lock);
125 rbnode = kctx->reg_rbtree.rb_node;
127 u64 tmp_start_pfn, tmp_end_pfn;
129 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
130 tmp_start_pfn = reg->start_pfn;
131 tmp_end_pfn = reg->start_pfn + reg->nr_pages;
133 /* If start is lower than this, go left. */
134 if (gpu_pfn < tmp_start_pfn)
135 rbnode = rbnode->rb_left;
136 /* If end is higher than this, then go right. */
137 else if (gpu_pfn >= tmp_end_pfn)
138 rbnode = rbnode->rb_right;
146 KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address);
148 /* Find region with given base address */
149 struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr)
151 u64 gpu_pfn = gpu_addr >> PAGE_SHIFT;
152 struct rb_node *rbnode;
153 struct kbase_va_region *reg;
155 KBASE_DEBUG_ASSERT(NULL != kctx);
157 lockdep_assert_held(&kctx->reg_lock);
159 rbnode = kctx->reg_rbtree.rb_node;
161 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
162 if (reg->start_pfn > gpu_pfn)
163 rbnode = rbnode->rb_left;
164 else if (reg->start_pfn < gpu_pfn)
165 rbnode = rbnode->rb_right;
174 KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address);
176 /* Find region meeting given requirements */
177 static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align)
179 struct rb_node *rbnode;
180 struct kbase_va_region *reg;
182 /* Note that this search is a linear search, as we do not have a target
183 address in mind, so does not benefit from the rbtree search */
184 rbnode = rb_first(&(kctx->reg_rbtree));
186 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
187 if ((reg->nr_pages >= nr_pages) &&
188 (reg->flags & KBASE_REG_FREE) &&
189 kbase_region_tracker_match_zone(reg, reg_reqs)) {
190 /* Check alignment */
191 u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1);
193 if ((start_pfn >= reg->start_pfn) &&
194 (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) &&
195 ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1)))
198 rbnode = rb_next(rbnode);
205 * @brief Remove a region object from the global list.
207 * The region reg is removed, possibly by merging with other free and
208 * compatible adjacent regions. It must be called with the context
209 * region lock held. The associated memory is not released (see
210 * kbase_free_alloced_region). Internal use only.
212 static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg)
214 struct rb_node *rbprev;
215 struct kbase_va_region *prev = NULL;
216 struct rb_node *rbnext;
217 struct kbase_va_region *next = NULL;
219 int merged_front = 0;
223 /* Try to merge with the previous block first */
224 rbprev = rb_prev(&(reg->rblink));
226 prev = rb_entry(rbprev, struct kbase_va_region, rblink);
227 if ((prev->flags & KBASE_REG_FREE) && kbase_region_tracker_match_zone(prev, reg)) {
228 /* We're compatible with the previous VMA, merge with it */
229 prev->nr_pages += reg->nr_pages;
230 rb_erase(&(reg->rblink), &kctx->reg_rbtree);
236 /* Try to merge with the next block second */
237 /* Note we do the lookup here as the tree may have been rebalanced. */
238 rbnext = rb_next(&(reg->rblink));
240 /* We're compatible with the next VMA, merge with it */
241 next = rb_entry(rbnext, struct kbase_va_region, rblink);
242 if ((next->flags & KBASE_REG_FREE) && kbase_region_tracker_match_zone(next, reg)) {
243 next->start_pfn = reg->start_pfn;
244 next->nr_pages += reg->nr_pages;
245 rb_erase(&(reg->rblink), &kctx->reg_rbtree);
248 /* We already merged with prev, free it */
249 kbase_free_alloced_region(reg);
254 /* If we failed to merge then we need to add a new block */
255 if (!(merged_front || merged_back)) {
257 * We didn't merge anything. Add a new free
258 * placeholder and remove the original one.
260 struct kbase_va_region *free_reg;
262 free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK);
268 rb_replace_node(&(reg->rblink), &(free_reg->rblink), &(kctx->reg_rbtree));
275 KBASE_EXPORT_TEST_API(kbase_remove_va_region);
278 * @brief Insert a VA region to the list, replacing the current at_reg.
280 static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages)
284 /* Must be a free region */
285 KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0);
286 /* start_pfn should be contained within at_reg */
287 KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages));
288 /* at least nr_pages from start_pfn should be contained within at_reg */
289 KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages);
291 new_reg->start_pfn = start_pfn;
292 new_reg->nr_pages = nr_pages;
294 /* Regions are a whole use, so swap and delete old one. */
295 if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) {
296 rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), &(kctx->reg_rbtree));
297 kbase_free_alloced_region(at_reg);
299 /* New region replaces the start of the old one, so insert before. */
300 else if (at_reg->start_pfn == start_pfn) {
301 at_reg->start_pfn += nr_pages;
302 KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages);
303 at_reg->nr_pages -= nr_pages;
305 kbase_region_tracker_insert(kctx, new_reg);
307 /* New region replaces the end of the old one, so insert after. */
308 else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) {
309 at_reg->nr_pages -= nr_pages;
311 kbase_region_tracker_insert(kctx, new_reg);
313 /* New region splits the old one, so insert and create new */
315 struct kbase_va_region *new_front_reg;
317 new_front_reg = kbase_alloc_free_region(kctx,
319 start_pfn - at_reg->start_pfn,
320 at_reg->flags & KBASE_REG_ZONE_MASK);
323 at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages;
324 at_reg->start_pfn = start_pfn + nr_pages;
326 kbase_region_tracker_insert(kctx, new_front_reg);
327 kbase_region_tracker_insert(kctx, new_reg);
337 * @brief Add a VA region to the list.
339 int kbase_add_va_region(struct kbase_context *kctx,
340 struct kbase_va_region *reg, u64 addr,
341 size_t nr_pages, size_t align)
343 struct kbase_va_region *tmp;
344 u64 gpu_pfn = addr >> PAGE_SHIFT;
347 KBASE_DEBUG_ASSERT(NULL != kctx);
348 KBASE_DEBUG_ASSERT(NULL != reg);
350 lockdep_assert_held(&kctx->reg_lock);
355 /* must be a power of 2 */
356 KBASE_DEBUG_ASSERT((align & (align - 1)) == 0);
357 KBASE_DEBUG_ASSERT(nr_pages > 0);
359 /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */
361 struct device *dev = kctx->kbdev->dev;
363 KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1)));
365 tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages);
367 dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages);
372 if ((!kbase_region_tracker_match_zone(tmp, reg)) ||
373 (!(tmp->flags & KBASE_REG_FREE))) {
374 dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK);
375 dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages);
376 dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align);
381 err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages);
383 dev_warn(dev, "Failed to insert va region");
391 /* Path 2: Map any free address which meets the requirements. */
395 tmp = kbase_region_tracker_find_region_meeting_reqs(kctx, reg, nr_pages, align);
400 start_pfn = (tmp->start_pfn + align - 1) & ~(align - 1);
401 err = kbase_insert_va_region_nolock(kctx, reg, tmp, start_pfn, nr_pages);
408 KBASE_EXPORT_TEST_API(kbase_add_va_region);
411 * @brief Initialize the internal region tracker data structure.
413 static void kbase_region_tracker_ds_init(struct kbase_context *kctx, struct kbase_va_region *same_va_reg, struct kbase_va_region *exec_reg, struct kbase_va_region *custom_va_reg)
415 kctx->reg_rbtree = RB_ROOT;
416 kbase_region_tracker_insert(kctx, same_va_reg);
418 /* exec and custom_va_reg doesn't always exist */
419 if (exec_reg && custom_va_reg) {
420 kbase_region_tracker_insert(kctx, exec_reg);
421 kbase_region_tracker_insert(kctx, custom_va_reg);
425 void kbase_region_tracker_term(struct kbase_context *kctx)
427 struct rb_node *rbnode;
428 struct kbase_va_region *reg;
431 rbnode = rb_first(&(kctx->reg_rbtree));
433 rb_erase(rbnode, &(kctx->reg_rbtree));
434 reg = rb_entry(rbnode, struct kbase_va_region, rblink);
435 kbase_free_alloced_region(reg);
441 * Initialize the region tracker data structure.
443 int kbase_region_tracker_init(struct kbase_context *kctx)
445 struct kbase_va_region *same_va_reg;
446 struct kbase_va_region *exec_reg = NULL;
447 struct kbase_va_region *custom_va_reg = NULL;
448 size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE;
449 u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE;
450 u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT;
452 #if defined(CONFIG_ARM64)
453 same_va_bits = VA_BITS;
454 #elif defined(CONFIG_X86_64)
456 #elif defined(CONFIG_64BIT)
457 #error Unsupported 64-bit architecture
463 else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA))
467 if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits)
470 /* all have SAME_VA */
471 same_va_reg = kbase_alloc_free_region(kctx, 1,
472 (1ULL << (same_va_bits - PAGE_SHIFT)) - 1,
473 KBASE_REG_ZONE_SAME_VA);
479 /* only 32-bit clients have the other two zones */
480 if (kctx->is_compat) {
482 if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) {
483 kbase_free_alloced_region(same_va_reg);
486 /* If the current size of TMEM is out of range of the
487 * virtual address space addressable by the MMU then
488 * we should shrink it to fit
490 if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit)
491 custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE;
493 exec_reg = kbase_alloc_free_region(kctx,
494 KBASE_REG_ZONE_EXEC_BASE,
495 KBASE_REG_ZONE_EXEC_SIZE,
496 KBASE_REG_ZONE_EXEC);
499 kbase_free_alloced_region(same_va_reg);
503 custom_va_reg = kbase_alloc_free_region(kctx,
504 KBASE_REG_ZONE_CUSTOM_VA_BASE,
505 custom_va_size, KBASE_REG_ZONE_CUSTOM_VA);
507 if (!custom_va_reg) {
508 kbase_free_alloced_region(same_va_reg);
509 kbase_free_alloced_region(exec_reg);
516 kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg);
521 int kbase_mem_init(struct kbase_device *kbdev)
523 struct kbasep_mem_device *memdev;
525 KBASE_DEBUG_ASSERT(kbdev);
527 memdev = &kbdev->memdev;
528 kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX;
530 /* Initialize memory usage */
531 atomic_set(&memdev->used_pages, 0);
533 return kbase_mem_pool_init(&kbdev->mem_pool,
534 KBASE_MEM_POOL_MAX_SIZE_KBDEV, kbdev, NULL);
537 void kbase_mem_halt(struct kbase_device *kbdev)
542 void kbase_mem_term(struct kbase_device *kbdev)
544 struct kbasep_mem_device *memdev;
547 KBASE_DEBUG_ASSERT(kbdev);
549 memdev = &kbdev->memdev;
551 pages = atomic_read(&memdev->used_pages);
553 dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
555 kbase_mem_pool_term(&kbdev->mem_pool);
558 KBASE_EXPORT_TEST_API(kbase_mem_term);
564 * @brief Allocate a free region object.
566 * The allocated object is not part of any list yet, and is flagged as
567 * KBASE_REG_FREE. No mapping is allocated yet.
569 * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC
572 struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone)
574 struct kbase_va_region *new_reg;
576 KBASE_DEBUG_ASSERT(kctx != NULL);
578 /* zone argument should only contain zone related region flags */
579 KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0);
580 KBASE_DEBUG_ASSERT(nr_pages > 0);
581 /* 64-bit address range is the max */
582 KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE));
584 new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL);
589 new_reg->cpu_alloc = NULL; /* no alloc bound yet */
590 new_reg->gpu_alloc = NULL; /* no alloc bound yet */
591 new_reg->kctx = kctx;
592 new_reg->flags = zone | KBASE_REG_FREE;
594 new_reg->flags |= KBASE_REG_GROWABLE;
596 new_reg->start_pfn = start_pfn;
597 new_reg->nr_pages = nr_pages;
602 KBASE_EXPORT_TEST_API(kbase_alloc_free_region);
605 * @brief Free a region object.
607 * The described region must be freed of any mapping.
609 * If the region is not flagged as KBASE_REG_FREE, the region's
610 * alloc object will be released.
611 * It is a bug if no alloc object exists for non-free regions.
614 void kbase_free_alloced_region(struct kbase_va_region *reg)
616 KBASE_DEBUG_ASSERT(NULL != reg);
617 if (!(reg->flags & KBASE_REG_FREE)) {
618 kbase_mem_phy_alloc_put(reg->cpu_alloc);
619 kbase_mem_phy_alloc_put(reg->gpu_alloc);
620 /* To detect use-after-free in debug builds */
621 KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE);
626 KBASE_EXPORT_TEST_API(kbase_free_alloced_region);
628 void kbase_mmu_update(struct kbase_context *kctx)
630 KBASE_DEBUG_ASSERT(NULL != kctx);
631 lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
632 /* ASSERT that the context has a valid as_nr, which is only the case
633 * when it's scheduled in.
635 * as_nr won't change because the caller has the runpool_irq lock */
636 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
637 lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex);
639 kctx->kbdev->mmu_mode->update(kctx);
642 KBASE_EXPORT_TEST_API(kbase_mmu_update);
644 void kbase_mmu_disable(struct kbase_context *kctx)
646 KBASE_DEBUG_ASSERT(NULL != kctx);
647 /* ASSERT that the context has a valid as_nr, which is only the case
648 * when it's scheduled in.
650 * as_nr won't change because the caller has the runpool_irq lock */
651 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
653 kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr);
656 KBASE_EXPORT_TEST_API(kbase_mmu_disable);
658 void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr)
660 kbdev->mmu_mode->disable_as(kbdev, as_nr);
663 int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align)
668 unsigned long mask = ~KBASE_REG_MEMATTR_MASK;
670 if ((kctx->kbdev->system_coherency == COHERENCY_ACE) &&
671 (reg->flags & KBASE_REG_SHARE_BOTH))
672 attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA);
674 attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC);
676 KBASE_DEBUG_ASSERT(NULL != kctx);
677 KBASE_DEBUG_ASSERT(NULL != reg);
679 err = kbase_add_va_region(kctx, reg, addr, nr_pages, align);
683 if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
685 struct kbase_mem_phy_alloc *alloc;
687 alloc = reg->gpu_alloc;
688 stride = alloc->imported.alias.stride;
689 KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased);
690 for (i = 0; i < alloc->imported.alias.nents; i++) {
691 if (alloc->imported.alias.aliased[i].alloc) {
692 err = kbase_mmu_insert_pages(kctx,
693 reg->start_pfn + (i * stride),
694 alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset,
695 alloc->imported.alias.aliased[i].length,
700 kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc);
702 err = kbase_mmu_insert_single_page(kctx,
703 reg->start_pfn + i * stride,
704 page_to_phys(kctx->aliasing_sink_page),
705 alloc->imported.alias.aliased[i].length,
706 (reg->flags & mask) | attr);
713 err = kbase_mmu_insert_pages(kctx, reg->start_pfn,
714 kbase_get_gpu_phy_pages(reg),
715 kbase_reg_current_backed_size(reg),
719 kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc);
725 if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
728 stride = reg->gpu_alloc->imported.alias.stride;
729 KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
731 if (reg->gpu_alloc->imported.alias.aliased[i].alloc) {
732 kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length);
733 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
737 kbase_remove_va_region(kctx, reg);
742 KBASE_EXPORT_TEST_API(kbase_gpu_mmap);
744 int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
748 if (reg->start_pfn == 0)
751 if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
754 err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages);
755 KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
756 for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++)
757 if (reg->gpu_alloc->imported.alias.aliased[i].alloc)
758 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
760 err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg));
761 kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc);
767 err = kbase_remove_va_region(kctx, reg);
771 static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping_of_region(const struct kbase_va_region *reg, unsigned long uaddr, size_t size)
773 struct kbase_cpu_mapping *map;
774 struct list_head *pos;
776 KBASE_DEBUG_ASSERT(NULL != reg);
777 KBASE_DEBUG_ASSERT(reg->cpu_alloc);
779 if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */
782 list_for_each(pos, ®->cpu_alloc->mappings) {
783 map = list_entry(pos, struct kbase_cpu_mapping, mappings_list);
784 if (map->vm_start <= uaddr && map->vm_end >= uaddr + size)
791 KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_of_region);
793 int kbasep_find_enclosing_cpu_mapping_offset(
794 struct kbase_context *kctx, u64 gpu_addr,
795 unsigned long uaddr, size_t size, u64 * offset)
797 struct kbase_cpu_mapping *map = NULL;
798 const struct kbase_va_region *reg;
801 KBASE_DEBUG_ASSERT(kctx != NULL);
803 kbase_gpu_vm_lock(kctx);
805 reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
806 if (reg && !(reg->flags & KBASE_REG_FREE)) {
807 map = kbasep_find_enclosing_cpu_mapping_of_region(reg, uaddr,
810 *offset = (uaddr - PTR_TO_U64(map->vm_start)) +
811 (map->page_off << PAGE_SHIFT);
816 kbase_gpu_vm_unlock(kctx);
821 KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset);
823 void kbase_sync_single(struct kbase_context *kctx,
824 phys_addr_t cpu_pa, phys_addr_t gpu_pa,
825 off_t offset, size_t size, enum kbase_sync_type sync_fn)
827 struct page *cpu_page;
829 cpu_page = pfn_to_page(PFN_DOWN(cpu_pa));
831 if (likely(cpu_pa == gpu_pa)) {
835 BUG_ON(offset + size > PAGE_SIZE);
837 dma_addr = kbase_dma_addr(cpu_page) + offset;
838 if (sync_fn == KBASE_SYNC_TO_CPU)
839 dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr,
840 size, DMA_BIDIRECTIONAL);
841 else if (sync_fn == KBASE_SYNC_TO_DEVICE)
842 dma_sync_single_for_device(kctx->kbdev->dev, dma_addr,
843 size, DMA_BIDIRECTIONAL);
847 struct page *gpu_page;
849 if (WARN(!gpu_pa, "No GPU PA found for infinite cache op"))
852 gpu_page = pfn_to_page(PFN_DOWN(gpu_pa));
854 if (sync_fn == KBASE_SYNC_TO_DEVICE) {
855 src = ((unsigned char *)kmap(cpu_page)) + offset;
856 dst = ((unsigned char *)kmap(gpu_page)) + offset;
857 } else if (sync_fn == KBASE_SYNC_TO_CPU) {
858 dma_sync_single_for_cpu(kctx->kbdev->dev,
859 kbase_dma_addr(gpu_page) + offset,
860 size, DMA_BIDIRECTIONAL);
861 src = ((unsigned char *)kmap(gpu_page)) + offset;
862 dst = ((unsigned char *)kmap(cpu_page)) + offset;
864 memcpy(dst, src, size);
867 if (sync_fn == KBASE_SYNC_TO_DEVICE)
868 dma_sync_single_for_device(kctx->kbdev->dev,
869 kbase_dma_addr(gpu_page) + offset,
870 size, DMA_BIDIRECTIONAL);
874 static int kbase_do_syncset(struct kbase_context *kctx,
875 struct base_syncset *set, enum kbase_sync_type sync_fn)
878 struct basep_syncset *sset = &set->basep_sset;
879 struct kbase_va_region *reg;
880 struct kbase_cpu_mapping *map;
885 u64 page_off, page_count;
889 kbase_os_mem_map_lock(kctx);
890 kbase_gpu_vm_lock(kctx);
892 /* find the region where the virtual address is contained */
893 reg = kbase_region_tracker_find_region_enclosing_address(kctx,
896 dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX",
902 if (!(reg->flags & KBASE_REG_CPU_CACHED))
905 start = (uintptr_t)sset->user_addr;
906 size = (size_t)sset->size;
908 map = kbasep_find_enclosing_cpu_mapping_of_region(reg, start, size);
910 dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX",
911 start, sset->mem_handle);
916 offset = start & (PAGE_SIZE - 1);
917 page_off = map->page_off + ((start - map->vm_start) >> PAGE_SHIFT);
918 page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
919 cpu_pa = kbase_get_cpu_phy_pages(reg);
920 gpu_pa = kbase_get_gpu_phy_pages(reg);
922 /* Sync first page */
923 if (cpu_pa[page_off]) {
924 size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
926 kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off],
927 offset, sz, sync_fn);
930 /* Sync middle pages (if any) */
931 for (i = 1; page_count > 2 && i < page_count - 1; i++) {
932 /* we grow upwards, so bail on first non-present page */
933 if (!cpu_pa[page_off + i])
936 kbase_sync_single(kctx, cpu_pa[page_off + i],
937 gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn);
940 /* Sync last page (if any) */
941 if (page_count > 1 && cpu_pa[page_off + page_count - 1]) {
942 size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1;
944 kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1],
945 gpu_pa[page_off + page_count - 1], 0, sz,
950 kbase_gpu_vm_unlock(kctx);
951 kbase_os_mem_map_unlock(kctx);
955 int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset)
958 struct basep_syncset *sset;
960 KBASE_DEBUG_ASSERT(NULL != kctx);
961 KBASE_DEBUG_ASSERT(NULL != syncset);
963 sset = &syncset->basep_sset;
965 switch (sset->type) {
966 case BASE_SYNCSET_OP_MSYNC:
967 err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_DEVICE);
970 case BASE_SYNCSET_OP_CSYNC:
971 err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_CPU);
975 dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type);
982 KBASE_EXPORT_TEST_API(kbase_sync_now);
984 /* vm lock must be held */
985 int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg)
989 KBASE_DEBUG_ASSERT(NULL != kctx);
990 KBASE_DEBUG_ASSERT(NULL != reg);
991 lockdep_assert_held(&kctx->reg_lock);
992 err = kbase_gpu_munmap(kctx, reg);
994 dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n");
997 #ifndef CONFIG_MALI_NO_MALI
998 if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) {
999 /* Wait for GPU to flush write buffer before freeing physical pages */
1000 kbase_wait_write_flush(kctx);
1003 /* This will also free the physical pages */
1004 kbase_free_alloced_region(reg);
1010 KBASE_EXPORT_TEST_API(kbase_mem_free_region);
1013 * @brief Free the region from the GPU and unregister it.
1015 * This function implements the free operation on a memory segment.
1016 * It will loudly fail if called with outstanding mappings.
1018 int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr)
1021 struct kbase_va_region *reg;
1023 KBASE_DEBUG_ASSERT(kctx != NULL);
1025 if (0 == gpu_addr) {
1026 dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n");
1029 kbase_gpu_vm_lock(kctx);
1031 if (gpu_addr >= BASE_MEM_COOKIE_BASE &&
1032 gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) {
1033 int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE);
1035 reg = kctx->pending_regions[cookie];
1041 /* ask to unlink the cookie as we'll free it */
1043 kctx->pending_regions[cookie] = NULL;
1044 kctx->cookies |= (1UL << cookie);
1046 kbase_free_alloced_region(reg);
1050 /* Validate the region */
1051 reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
1052 if (!reg || (reg->flags & KBASE_REG_FREE)) {
1053 dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX",
1059 if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) {
1060 /* SAME_VA must be freed through munmap */
1061 dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__,
1067 err = kbase_mem_free_region(kctx, reg);
1071 kbase_gpu_vm_unlock(kctx);
1075 KBASE_EXPORT_TEST_API(kbase_mem_free);
1077 void kbase_update_region_flags(struct kbase_context *kctx,
1078 struct kbase_va_region *reg, unsigned long flags)
1080 KBASE_DEBUG_ASSERT(NULL != reg);
1081 KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0);
1083 reg->flags |= kbase_cache_enabled(flags, reg->nr_pages);
1084 /* all memory is now growable */
1085 reg->flags |= KBASE_REG_GROWABLE;
1087 if (flags & BASE_MEM_GROW_ON_GPF)
1088 reg->flags |= KBASE_REG_PF_GROW;
1090 if (flags & BASE_MEM_PROT_CPU_WR)
1091 reg->flags |= KBASE_REG_CPU_WR;
1093 if (flags & BASE_MEM_PROT_CPU_RD)
1094 reg->flags |= KBASE_REG_CPU_RD;
1096 if (flags & BASE_MEM_PROT_GPU_WR)
1097 reg->flags |= KBASE_REG_GPU_WR;
1099 if (flags & BASE_MEM_PROT_GPU_RD)
1100 reg->flags |= KBASE_REG_GPU_RD;
1102 if (0 == (flags & BASE_MEM_PROT_GPU_EX))
1103 reg->flags |= KBASE_REG_GPU_NX;
1105 if (flags & BASE_MEM_COHERENT_SYSTEM ||
1106 flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED)
1107 reg->flags |= KBASE_REG_SHARE_BOTH;
1108 else if (flags & BASE_MEM_COHERENT_LOCAL)
1109 reg->flags |= KBASE_REG_SHARE_IN;
1111 /* Set up default MEMATTR usage */
1112 if (kctx->kbdev->system_coherency == COHERENCY_ACE &&
1113 (reg->flags & KBASE_REG_SHARE_BOTH)) {
1115 KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE);
1118 KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT);
1121 KBASE_EXPORT_TEST_API(kbase_update_region_flags);
1123 int kbase_alloc_phy_pages_helper(
1124 struct kbase_mem_phy_alloc *alloc,
1125 size_t nr_pages_requested)
1127 KBASE_DEBUG_ASSERT(alloc);
1128 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE);
1129 KBASE_DEBUG_ASSERT(alloc->imported.kctx);
1131 if (nr_pages_requested == 0)
1132 goto done; /*nothing to do*/
1134 kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->used_pages);
1135 kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages);
1137 /* Increase mm counters before we allocate pages so that this
1138 * allocation is visible to the OOM killer */
1139 kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested);
1141 if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool,
1142 nr_pages_requested, alloc->pages + alloc->nents) != 0)
1145 #if defined(CONFIG_MALI_MIPE_ENABLED)
1146 kbase_tlstream_aux_pagesalloc((s64)nr_pages_requested);
1149 alloc->nents += nr_pages_requested;
1154 kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_requested);
1155 kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->used_pages);
1156 kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages);
1161 int kbase_free_phy_pages_helper(
1162 struct kbase_mem_phy_alloc *alloc,
1163 size_t nr_pages_to_free)
1166 phys_addr_t *start_free;
1168 KBASE_DEBUG_ASSERT(alloc);
1169 KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE);
1170 KBASE_DEBUG_ASSERT(alloc->imported.kctx);
1171 KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free);
1173 /* early out if nothing to do */
1174 if (0 == nr_pages_to_free)
1177 start_free = alloc->pages + alloc->nents - nr_pages_to_free;
1179 syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
1181 kbase_mem_pool_free_pages(&alloc->imported.kctx->mem_pool,
1186 alloc->nents -= nr_pages_to_free;
1187 kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_to_free);
1188 kbase_atomic_sub_pages(nr_pages_to_free, &alloc->imported.kctx->used_pages);
1189 kbase_atomic_sub_pages(nr_pages_to_free, &alloc->imported.kctx->kbdev->memdev.used_pages);
1191 #if defined(CONFIG_MALI_MIPE_ENABLED)
1192 kbase_tlstream_aux_pagesalloc(-(s64)nr_pages_to_free);
1198 void kbase_mem_kref_free(struct kref *kref)
1200 struct kbase_mem_phy_alloc *alloc;
1202 alloc = container_of(kref, struct kbase_mem_phy_alloc, kref);
1204 switch (alloc->type) {
1205 case KBASE_MEM_TYPE_NATIVE: {
1206 KBASE_DEBUG_ASSERT(alloc->imported.kctx);
1207 kbase_free_phy_pages_helper(alloc, alloc->nents);
1210 case KBASE_MEM_TYPE_ALIAS: {
1211 /* just call put on the underlying phy allocs */
1213 struct kbase_aliased *aliased;
1215 aliased = alloc->imported.alias.aliased;
1217 for (i = 0; i < alloc->imported.alias.nents; i++)
1218 if (aliased[i].alloc)
1219 kbase_mem_phy_alloc_put(aliased[i].alloc);
1224 case KBASE_MEM_TYPE_RAW:
1225 /* raw pages, external cleanup */
1228 case KBASE_MEM_TYPE_IMPORTED_UMP:
1229 ump_dd_release(alloc->imported.ump_handle);
1232 #ifdef CONFIG_DMA_SHARED_BUFFER
1233 case KBASE_MEM_TYPE_IMPORTED_UMM:
1234 dma_buf_detach(alloc->imported.umm.dma_buf,
1235 alloc->imported.umm.dma_attachment);
1236 dma_buf_put(alloc->imported.umm.dma_buf);
1239 case KBASE_MEM_TYPE_IMPORTED_USER_BUF:
1240 kfree(alloc->imported.user_buf.pages);
1242 case KBASE_MEM_TYPE_TB:{
1245 tb = alloc->imported.kctx->jctx.tb;
1246 kbase_device_trace_buffer_uninstall(alloc->imported.kctx);
1251 WARN(1, "Unexecpted free of type %d\n", alloc->type);
1255 /* Free based on allocation type */
1256 if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE)
1262 KBASE_EXPORT_TEST_API(kbase_mem_kref_free);
1264 int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size)
1266 KBASE_DEBUG_ASSERT(NULL != reg);
1267 KBASE_DEBUG_ASSERT(vsize > 0);
1269 /* validate user provided arguments */
1270 if (size > vsize || vsize > reg->nr_pages)
1273 /* Prevent vsize*sizeof from wrapping around.
1274 * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail.
1276 if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages)))
1279 KBASE_DEBUG_ASSERT(0 != vsize);
1281 if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0)
1284 if (reg->cpu_alloc != reg->gpu_alloc) {
1285 if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0)
1292 kbase_free_phy_pages_helper(reg->cpu_alloc, size);
1297 KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages);
1299 bool kbase_check_alloc_flags(unsigned long flags)
1301 /* Only known input flags should be set. */
1302 if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
1305 /* At least one flag should be set */
1309 /* Either the GPU or CPU must be reading from the allocated memory */
1310 if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0)
1313 /* Either the GPU or CPU must be writing to the allocated memory */
1314 if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0)
1317 /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */
1318 if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF)))
1321 /* GPU should have at least read or write access otherwise there is no
1322 reason for allocating. */
1323 if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
1329 bool kbase_check_import_flags(unsigned long flags)
1331 /* Only known input flags should be set. */
1332 if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
1335 /* At least one flag should be set */
1339 /* Imported memory cannot be GPU executable */
1340 if (flags & BASE_MEM_PROT_GPU_EX)
1343 /* Imported memory cannot grow on page fault */
1344 if (flags & BASE_MEM_GROW_ON_GPF)
1347 /* GPU should have at least read or write access otherwise there is no
1348 reason for importing. */
1349 if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
1352 /* Secure memory cannot be read by the CPU */
1353 if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD))
1360 * @brief Acquire the per-context region list lock
1362 void kbase_gpu_vm_lock(struct kbase_context *kctx)
1364 KBASE_DEBUG_ASSERT(kctx != NULL);
1365 mutex_lock(&kctx->reg_lock);
1368 KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock);
1371 * @brief Release the per-context region list lock
1373 void kbase_gpu_vm_unlock(struct kbase_context *kctx)
1375 KBASE_DEBUG_ASSERT(kctx != NULL);
1376 mutex_unlock(&kctx->reg_lock);
1379 KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock);