drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c

   1 /*
   2  * Copyright (C) 2013-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include <linux/list.h>
  12 #include <linux/mm.h>
  13 #include <linux/mm_types.h>
  14 #include <linux/fs.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/slab.h>
  17 #include <linux/version.h>
  18 #include <linux/sched.h>
  19 #include <linux/idr.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/shmem_fs.h>
  23 #include <linux/file.h>
  24 #include <linux/swap.h>
  25 #include <linux/pagemap.h>
  26 #include "mali_osk.h"
  27 #include "mali_osk_mali.h"
  28 #include "mali_memory.h"
  29 #include "mali_memory_manager.h"
  30 #include "mali_memory_virtual.h"
  31 #include "mali_memory_cow.h"
  32 #include "mali_ukk.h"
  33 #include "mali_kernel_utilization.h"
  34 #include "mali_memory_swap_alloc.h"
  35
  36
  37 static struct _mali_osk_bitmap idx_mgr;
  38 static struct file *global_swap_file;
  39 static struct address_space *global_swap_space;
  40 static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL;
  41 static u32 mem_backend_swapped_pool_size;
  42 #ifdef MALI_MEM_SWAP_TRACKING
  43 static u32 mem_backend_swapped_unlock_size;
  44 #endif
  45 /* Lock order: mem_backend_swapped_pool_lock  > each memory backend's mutex lock.
  46  * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */
  47 static struct mutex mem_backend_swapped_pool_lock;
  48 static struct list_head mem_backend_swapped_pool;
  49
  50 extern struct mali_mem_os_allocator mali_mem_os_allocator;
  51
  52 #define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024)
  53 #define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0)               /* Used to mark the given memory cookie is invalidate. */
  54 #define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF)
  55 #define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_CACHE_SHIFT)
  56 #define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */
  57
  58 unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE;
  59
  60 /**
  61  * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too
  62  * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed
  63  * the threshold value of the swapped pool size.
  64  */
  65 typedef enum {
  66         MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100,
  67         MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257,
  68 } _mali_mem_swap_pool_shrink_type_t;
  69
  70 static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg);
  71
  72 _mali_osk_errcode_t mali_mem_swap_init(void)
  73 {
  74         gfp_t flags = __GFP_NORETRY | __GFP_NOWARN;
  75
  76         if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) {
  77                 return _MALI_OSK_ERR_NOMEM;
  78         }
  79
  80         global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE);
  81         if (IS_ERR(global_swap_file)) {
  82                 _mali_osk_bitmap_term(&idx_mgr);
  83                 return _MALI_OSK_ERR_NOMEM;
  84         }
  85
  86         global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping;
  87
  88         mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL);
  89         if (NULL == mali_mem_swap_out_workq) {
  90                 _mali_osk_bitmap_term(&idx_mgr);
  91                 fput(global_swap_file);
  92                 return _MALI_OSK_ERR_NOMEM;
  93         }
  94
  95 #if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE)
  96         flags |= GFP_HIGHUSER;
  97 #else
  98 #ifdef CONFIG_ZONE_DMA32
  99         flags |= GFP_DMA32;
 100 #else
 101 #ifdef CONFIG_ZONE_DMA
 102         flags |= GFP_DMA;
 103 #else
 104         /* arm64 utgard only work on < 4G, but the kernel
 105          * didn't provide method to allocte memory < 4G
 106          */
 107         MALI_DEBUG_ASSERT(0);
 108 #endif
 109 #endif
 110 #endif
 111
 112         /* When we use shmem_read_mapping_page to allocate/swap-in, it will
 113          * use these flags to allocate new page if need.*/
 114         mapping_set_gfp_mask(global_swap_space, flags);
 115
 116         mem_backend_swapped_pool_size = 0;
 117 #ifdef MALI_MEM_SWAP_TRACKING
 118         mem_backend_swapped_unlock_size = 0;
 119 #endif
 120         mutex_init(&mem_backend_swapped_pool_lock);
 121         INIT_LIST_HEAD(&mem_backend_swapped_pool);
 122
 123         MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20));
 124
 125         return _MALI_OSK_ERR_OK;
 126 }
 127
 128 void mali_mem_swap_term(void)
 129 {
 130         _mali_osk_bitmap_term(&idx_mgr);
 131
 132         fput(global_swap_file);
 133
 134         _mali_osk_wq_delete_work(mali_mem_swap_out_workq);
 135
 136         MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool));
 137         MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size);
 138
 139         return;
 140 }
 141
 142 struct file *mali_mem_swap_get_global_swap_file(void)
 143 {
 144         return  global_swap_file;
 145 }
 146
 147 /* Judge if swappable backend in swapped pool. */
 148 static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend)
 149 {
 150         MALI_DEBUG_ASSERT_POINTER(mem_bkend);
 151
 152         return !list_empty(&mem_bkend->list);
 153 }
 154
 155 void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend)
 156 {
 157         MALI_DEBUG_ASSERT_POINTER(mem_bkend);
 158
 159         mutex_lock(&mem_backend_swapped_pool_lock);
 160         mutex_lock(&mem_bkend->mutex);
 161
 162         if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) {
 163                 mutex_unlock(&mem_bkend->mutex);
 164                 mutex_unlock(&mem_backend_swapped_pool_lock);
 165                 return;
 166         }
 167
 168         MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list));
 169
 170         list_del_init(&mem_bkend->list);
 171
 172         mutex_unlock(&mem_bkend->mutex);
 173
 174         mem_backend_swapped_pool_size -= mem_bkend->size;
 175
 176         mutex_unlock(&mem_backend_swapped_pool_lock);
 177 }
 178
 179 static void mali_mem_swap_out_page_node(mali_page_node *page_node)
 180 {
 181         MALI_DEBUG_ASSERT(page_node);
 182
 183         dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr,
 184                        _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
 185         set_page_dirty(page_node->swap_it->page);
 186         page_cache_release(page_node->swap_it->page);
 187 }
 188
 189 void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend)
 190 {
 191         mali_page_node *m_page;
 192
 193         MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex));
 194
 195         if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) {
 196                 return;
 197         }
 198
 199         mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN;
 200
 201         list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
 202                 mali_mem_swap_out_page_node(m_page);
 203         }
 204
 205         return;
 206 }
 207
 208 static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node)
 209 {
 210         mali_page_node *m_page;
 211
 212         MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex));
 213
 214         list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
 215                 if (m_page == page_node) {
 216                         break;
 217                 }
 218                 mali_mem_swap_out_page_node(m_page);
 219         }
 220 }
 221
 222 static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type)
 223 {
 224         mali_mem_backend *bkend, *tmp_bkend;
 225         long system_free_size;
 226         u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value;
 227
 228         MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock));
 229
 230         if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) {
 231                 /**
 232                  * When we met that system memory is very low and Mali locked swappable memory size is less than
 233                  * threshold value, and at the same time, GPU load is very low and don't need high performance,
 234                  * at this condition, we can unlock more swap memory backend from swapped backends pool.
 235                  */
 236                 gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION;
 237                 temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2);
 238         } else {
 239                 /* When we add swappable memory backends to swapped pool, we need to think that we couldn't
 240                 * hold too much swappable backends in Mali driver, and also we need considering performance.
 241                 * So there is a balance for swapping out memory backend, we should follow the following conditions:
 242                 * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value.
 243                 * 2. System level free memory size is less than the defined threshold value.
 244                 * 3. Please note that GPU utilization problem isn't considered in this condition.
 245                 */
 246                 gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS;
 247                 temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value;
 248         }
 249
 250         /* Get system free pages number. */
 251         system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE;
 252         last_gpu_utilization = _mali_ukk_utilization_gp_pp();
 253
 254         if ((last_gpu_utilization < gpu_utilization_threshold_value)
 255             && (system_free_size < mali_mem_swap_out_threshold_value)
 256             && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) {
 257                 list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) {
 258                         if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) {
 259                                 break;
 260                         }
 261
 262                         mutex_lock(&bkend->mutex);
 263
 264                         /* check if backend is in use. */
 265                         if (0 < bkend->using_count) {
 266                                 mutex_unlock(&bkend->mutex);
 267                                 continue;
 268                         }
 269
 270                         mali_mem_swap_unlock_single_mem_backend(bkend);
 271                         list_del_init(&bkend->list);
 272                         mem_backend_swapped_pool_size -= bkend->size;
 273 #ifdef MALI_MEM_SWAP_TRACKING
 274                         mem_backend_swapped_unlock_size += bkend->size;
 275 #endif
 276                         mutex_unlock(&bkend->mutex);
 277                 }
 278         }
 279
 280         return;
 281 }
 282
 283 static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg)
 284 {
 285         MALI_IGNORE(arg);
 286
 287         mutex_lock(&mem_backend_swapped_pool_lock);
 288
 289         mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION);
 290
 291         mutex_unlock(&mem_backend_swapped_pool_lock);
 292 }
 293
 294 /**
 295  * After PP job finished, we add all of swappable memory backend used by this PP
 296  * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold
 297  * value, we also need to shrink the swapped pool start from the head of the list.
 298  */
 299 void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend)
 300 {
 301         mutex_lock(&mem_backend_swapped_pool_lock);
 302         mutex_lock(&mem_bkend->mutex);
 303
 304         if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) {
 305                 MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list));
 306
 307                 list_del_init(&mem_bkend->list);
 308                 list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool);
 309                 mutex_unlock(&mem_bkend->mutex);
 310                 mutex_unlock(&mem_backend_swapped_pool_lock);
 311                 return;
 312         }
 313
 314         list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool);
 315
 316         mutex_unlock(&mem_bkend->mutex);
 317         mem_backend_swapped_pool_size += mem_bkend->size;
 318
 319         mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS);
 320
 321         mutex_unlock(&mem_backend_swapped_pool_lock);
 322         return;
 323 }
 324
 325
 326 u32 mali_mem_swap_idx_alloc(void)
 327 {
 328         return _mali_osk_bitmap_alloc(&idx_mgr);
 329 }
 330
 331 void mali_mem_swap_idx_free(u32 idx)
 332 {
 333         _mali_osk_bitmap_free(&idx_mgr, idx);
 334 }
 335
 336 static u32 mali_mem_swap_idx_range_alloc(u32 count)
 337 {
 338         u32 index;
 339
 340         index = _mali_osk_bitmap_alloc_range(&idx_mgr, count);
 341
 342         return index;
 343 }
 344
 345 static void mali_mem_swap_idx_range_free(u32 idx, int num)
 346 {
 347         _mali_osk_bitmap_free_range(&idx_mgr, idx, num);
 348 }
 349
 350 struct mali_swap_item *mali_mem_swap_alloc_swap_item(void)
 351 {
 352         mali_swap_item *swap_item;
 353
 354         swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL);
 355
 356         if (NULL == swap_item) {
 357                 return NULL;
 358         }
 359
 360         atomic_set(&swap_item->ref_count, 1);
 361         swap_item->page = NULL;
 362         atomic_add(1, &mali_mem_os_allocator.allocated_pages);
 363
 364         return swap_item;
 365 }
 366
 367 void mali_mem_swap_free_swap_item(mali_swap_item *swap_item)
 368 {
 369         struct inode *file_node;
 370         long long start, end;
 371
 372         /* If this swap item is shared, we just reduce the reference counter. */
 373         if (0 == atomic_dec_return(&swap_item->ref_count)) {
 374                 file_node = global_swap_file->f_path.dentry->d_inode;
 375                 start = swap_item->idx;
 376                 start = start << 12;
 377                 end = start + PAGE_SIZE;
 378
 379                 shmem_truncate_range(file_node, start, (end - 1));
 380
 381                 mali_mem_swap_idx_free(swap_item->idx);
 382
 383                 atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
 384
 385                 kfree(swap_item);
 386         }
 387 }
 388
 389 /* Used to allocate new swap item for new memory allocation and cow page for write. */
 390 struct mali_page_node *_mali_mem_swap_page_node_allocate(void)
 391 {
 392         struct mali_page_node *m_page;
 393
 394         m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP);
 395
 396         if (NULL == m_page) {
 397                 return NULL;
 398         }
 399
 400         m_page->swap_it = mali_mem_swap_alloc_swap_item();
 401
 402         if (NULL == m_page->swap_it) {
 403                 kfree(m_page);
 404                 return NULL;
 405         }
 406
 407         return m_page;
 408 }
 409
 410 _mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page)
 411 {
 412
 413         mali_mem_swap_free_swap_item(m_page->swap_it);
 414
 415         return _MALI_OSK_ERR_OK;
 416 }
 417
 418 void _mali_mem_swap_page_node_free(struct mali_page_node *m_page)
 419 {
 420         _mali_mem_swap_put_page_node(m_page);
 421
 422         kfree(m_page);
 423
 424         return;
 425 }
 426
 427 u32 mali_mem_swap_free(mali_mem_swap *swap_mem)
 428 {
 429         struct mali_page_node *m_page, *m_tmp;
 430         u32 free_pages_nr = 0;
 431
 432         MALI_DEBUG_ASSERT_POINTER(swap_mem);
 433
 434         list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) {
 435                 MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP);
 436
 437                 /* free the page node and release the swap item, if the ref count is 1,
 438                  * then need also free the swap item. */
 439                 list_del(&m_page->list);
 440                 if (1 == _mali_page_node_get_ref_count(m_page)) {
 441                         free_pages_nr++;
 442                 }
 443
 444                 _mali_mem_swap_page_node_free(m_page);
 445         }
 446
 447         return free_pages_nr;
 448 }
 449
 450 static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem)
 451 {
 452         struct mali_page_node *m_page, *m_tmp;
 453         u32 free_pages_nr = 0;
 454
 455         MALI_DEBUG_ASSERT_POINTER(cow_mem);
 456
 457         list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) {
 458                 MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP);
 459
 460                 /* free the page node and release the swap item, if the ref count is 1,
 461                  * then need also free the swap item. */
 462                 list_del(&m_page->list);
 463                 if (1 == _mali_page_node_get_ref_count(m_page)) {
 464                         free_pages_nr++;
 465                 }
 466
 467                 _mali_mem_swap_page_node_free(m_page);
 468         }
 469
 470         return free_pages_nr;
 471 }
 472
 473 u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped)
 474 {
 475         mali_mem_allocation *alloc;
 476         u32 free_pages_nr = 0;
 477
 478         MALI_DEBUG_ASSERT_POINTER(mem_bkend);
 479         alloc = mem_bkend->mali_allocation;
 480         MALI_DEBUG_ASSERT_POINTER(alloc);
 481
 482         if (is_mali_mapped) {
 483                 mali_mem_swap_mali_unmap(alloc);
 484         }
 485
 486         mali_memory_swap_list_backend_delete(mem_bkend);
 487
 488         mutex_lock(&mem_bkend->mutex);
 489         /* To make sure the given memory backend was unlocked from Mali side,
 490          * and then free this memory block. */
 491         mali_mem_swap_unlock_single_mem_backend(mem_bkend);
 492         mutex_unlock(&mem_bkend->mutex);
 493
 494         if (MALI_MEM_SWAP == mem_bkend->type) {
 495                 free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem);
 496         } else {
 497                 free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem);
 498         }
 499
 500         return free_pages_nr;
 501 }
 502
 503 mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node)
 504 {
 505         MALI_DEBUG_ASSERT(NULL != page_node);
 506
 507         page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx);
 508
 509         if (IS_ERR(page_node->swap_it->page)) {
 510                 MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx));
 511                 return MALI_FALSE;
 512         }
 513
 514         /* Ensure page is flushed from CPU caches. */
 515         page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page,
 516                                        0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
 517
 518         return MALI_TRUE;
 519 }
 520
 521 int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx)
 522 {
 523         size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE;
 524         struct mali_page_node *m_page;
 525         long system_free_size;
 526         u32 i, index;
 527         mali_bool ret;
 528
 529         MALI_DEBUG_ASSERT(NULL != swap_mem);
 530         MALI_DEBUG_ASSERT(NULL != bkend_idx);
 531         MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE);
 532
 533         if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) {
 534                 MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n",
 535                                      size,
 536                                      atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE,
 537                                      mali_mem_os_allocator.allocation_limit));
 538                 return _MALI_OSK_ERR_NOMEM;
 539         }
 540
 541         INIT_LIST_HEAD(&swap_mem->pages);
 542         swap_mem->count = page_count;
 543         index = mali_mem_swap_idx_range_alloc(page_count);
 544
 545         if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) {
 546                 MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory."));
 547                 return _MALI_OSK_ERR_FAULT;
 548         }
 549
 550         for (i = 0; i < page_count; i++) {
 551                 m_page = _mali_mem_swap_page_node_allocate();
 552
 553                 if (NULL == m_page) {
 554                         MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node."));
 555                         swap_mem->count = i;
 556
 557                         mali_mem_swap_free(swap_mem);
 558                         mali_mem_swap_idx_range_free(index + i, page_count - i);
 559                         return _MALI_OSK_ERR_FAULT;
 560                 }
 561
 562                 m_page->swap_it->idx = index + i;
 563
 564                 ret = mali_mem_swap_in_page_node(m_page);
 565
 566                 if (MALI_FALSE == ret) {
 567                         MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed."));
 568                         _mali_mem_swap_page_node_free(m_page);
 569                         mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1);
 570
 571                         swap_mem->count = i;
 572                         mali_mem_swap_free(swap_mem);
 573                         return _MALI_OSK_ERR_NOMEM;
 574                 }
 575
 576                 list_add_tail(&m_page->list, &swap_mem->pages);
 577         }
 578
 579         system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE;
 580
 581         if ((system_free_size < mali_mem_swap_out_threshold_value)
 582             && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2))
 583             && mali_utilization_enabled()) {
 584                 _mali_osk_wq_schedule_work(mali_mem_swap_out_workq);
 585         }
 586
 587         *bkend_idx = index;
 588         return 0;
 589 }
 590
 591 void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc)
 592 {
 593         struct mali_session_data *session;
 594
 595         MALI_DEBUG_ASSERT_POINTER(alloc);
 596         session = alloc->session;
 597         MALI_DEBUG_ASSERT_POINTER(session);
 598
 599         mali_session_memory_lock(session);
 600         mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
 601                                alloc->flags);
 602         mali_session_memory_unlock(session);
 603 }
 604
 605
 606 /* Insert these pages from shmem to mali page table*/
 607 _mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props)
 608 {
 609         struct mali_page_directory *pagedir = session->page_directory;
 610         struct mali_page_node *m_page;
 611         dma_addr_t phys;
 612         u32 virt = vaddr;
 613         u32 prop = props;
 614
 615         list_for_each_entry(m_page, &swap_mem->pages, list) {
 616                 MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page);
 617                 phys = m_page->swap_it->dma_addr;
 618
 619                 mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop);
 620                 virt += MALI_MMU_PAGE_SIZE;
 621         }
 622
 623         return _MALI_OSK_ERR_OK;
 624 }
 625
 626 int mali_mem_swap_in_pages(struct mali_pp_job *job)
 627 {
 628         u32 num_memory_cookies;
 629         struct mali_session_data *session;
 630         struct mali_vma_node *mali_vma_node = NULL;
 631         mali_mem_allocation *mali_alloc = NULL;
 632         mali_mem_backend *mem_bkend = NULL;
 633         struct mali_page_node *m_page;
 634         mali_bool swap_in_success = MALI_TRUE;
 635         int i;
 636
 637         MALI_DEBUG_ASSERT_POINTER(job);
 638
 639         num_memory_cookies = mali_pp_job_num_memory_cookies(job);
 640         session = mali_pp_job_get_session(job);
 641
 642         MALI_DEBUG_ASSERT_POINTER(session);
 643
 644         for (i = 0; i < num_memory_cookies; i++) {
 645
 646                 u32 mali_addr  = mali_pp_job_get_memory_cookie(job, i);
 647
 648                 mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
 649                 if (NULL == mali_vma_node) {
 650                         job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
 651                         swap_in_success = MALI_FALSE;
 652                         MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr));
 653                         continue;
 654                 }
 655
 656                 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
 657                 MALI_DEBUG_ASSERT(NULL != mali_alloc);
 658
 659                 if (MALI_MEM_SWAP != mali_alloc->type &&
 660                     MALI_MEM_COW != mali_alloc->type) {
 661                         continue;
 662                 }
 663
 664                 /* Get backend memory & Map on GPU */
 665                 mutex_lock(&mali_idr_mutex);
 666                 mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
 667                 mutex_unlock(&mali_idr_mutex);
 668                 MALI_DEBUG_ASSERT(NULL != mem_bkend);
 669
 670                 /* We neednot hold backend's lock here, race safe.*/
 671                 if ((MALI_MEM_COW == mem_bkend->type) &&
 672                     (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
 673                         continue;
 674                 }
 675
 676                 mutex_lock(&mem_bkend->mutex);
 677
 678                 /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in,
 679                  * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which
 680                  * should not be swapped out when delete job to invalide */
 681                 if (MALI_FALSE == swap_in_success) {
 682                         job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
 683                         mutex_unlock(&mem_bkend->mutex);
 684                         continue;
 685                 }
 686
 687                 /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */
 688                 ++mem_bkend->using_count;
 689
 690                 if (1 < mem_bkend->using_count) {
 691                         MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags));
 692                         mutex_unlock(&mem_bkend->mutex);
 693                         continue;
 694                 }
 695
 696                 if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) {
 697                         mutex_unlock(&mem_bkend->mutex);
 698                         continue;
 699                 }
 700
 701
 702                 list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
 703                         if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) {
 704                                 /* Don't have enough memory to swap in page, so release pages have already been swapped
 705                                  * in and then mark this pp job to be fail. */
 706                                 mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page);
 707                                 swap_in_success = MALI_FALSE;
 708                                 break;
 709                         }
 710                 }
 711
 712                 if (swap_in_success) {
 713 #ifdef MALI_MEM_SWAP_TRACKING
 714                         mem_backend_swapped_unlock_size -= mem_bkend->size;
 715 #endif
 716                         _mali_osk_mutex_wait(session->memory_lock);
 717                         mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties);
 718                         _mali_osk_mutex_signal(session->memory_lock);
 719
 720                         /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */
 721                         mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN);
 722                         mutex_unlock(&mem_bkend->mutex);
 723                 } else {
 724                         --mem_bkend->using_count;
 725                         /* Marking that this backend is not swapped in, need not to be processed anymore. */
 726                         job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
 727                         mutex_unlock(&mem_bkend->mutex);
 728                 }
 729         }
 730
 731         job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL;
 732
 733         return _MALI_OSK_ERR_OK;
 734 }
 735
 736 int mali_mem_swap_out_pages(struct mali_pp_job *job)
 737 {
 738         u32 num_memory_cookies;
 739         struct mali_session_data *session;
 740         struct mali_vma_node *mali_vma_node = NULL;
 741         mali_mem_allocation *mali_alloc = NULL;
 742         mali_mem_backend *mem_bkend = NULL;
 743         int i;
 744
 745         MALI_DEBUG_ASSERT_POINTER(job);
 746
 747         num_memory_cookies = mali_pp_job_num_memory_cookies(job);
 748         session = mali_pp_job_get_session(job);
 749
 750         MALI_DEBUG_ASSERT_POINTER(session);
 751
 752
 753         for (i = 0; i < num_memory_cookies; i++) {
 754                 u32 mali_addr  = mali_pp_job_get_memory_cookie(job, i);
 755
 756                 if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) {
 757                         continue;
 758                 }
 759
 760                 mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
 761
 762                 if (NULL == mali_vma_node) {
 763                         MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr));
 764                         continue;
 765                 }
 766
 767                 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
 768                 MALI_DEBUG_ASSERT(NULL != mali_alloc);
 769
 770                 if (MALI_MEM_SWAP != mali_alloc->type &&
 771                     MALI_MEM_COW != mali_alloc->type) {
 772                         continue;
 773                 }
 774
 775                 mutex_lock(&mali_idr_mutex);
 776                 mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
 777                 mutex_unlock(&mali_idr_mutex);
 778                 MALI_DEBUG_ASSERT(NULL != mem_bkend);
 779
 780                 /* We neednot hold backend's lock here, race safe.*/
 781                 if ((MALI_MEM_COW == mem_bkend->type) &&
 782                     (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
 783                         continue;
 784                 }
 785
 786                 mutex_lock(&mem_bkend->mutex);
 787
 788                 MALI_DEBUG_ASSERT(0 < mem_bkend->using_count);
 789
 790                 /* Reducing the using_count of mem backend means less pp job are using this memory backend,
 791                  * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */
 792                 --mem_bkend->using_count;
 793
 794                 if (0 < mem_bkend->using_count) {
 795                         mutex_unlock(&mem_bkend->mutex);
 796                         continue;
 797                 }
 798                 mutex_unlock(&mem_bkend->mutex);
 799
 800                 mali_memory_swap_list_backend_add(mem_bkend);
 801         }
 802
 803         return _MALI_OSK_ERR_OK;
 804 }
 805
 806 int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep)
 807 {
 808         struct mali_page_node *m_page, *found_node = NULL;
 809         struct page *found_page;
 810         mali_mem_swap *swap = NULL;
 811         mali_mem_cow *cow = NULL;
 812         dma_addr_t dma_addr;
 813         u32 i = 0;
 814
 815         if (MALI_MEM_SWAP == mem_bkend->type) {
 816                 swap = &mem_bkend->swap_mem;
 817                 list_for_each_entry(m_page, &swap->pages, list) {
 818                         if (i == offset) {
 819                                 found_node = m_page;
 820                                 break;
 821                         }
 822                         i++;
 823                 }
 824         } else {
 825                 MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
 826                 MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags));
 827
 828                 cow = &mem_bkend->cow_mem;
 829                 list_for_each_entry(m_page, &cow->pages, list) {
 830                         if (i == offset) {
 831                                 found_node = m_page;
 832                                 break;
 833                         }
 834                         i++;
 835                 }
 836         }
 837
 838         if (NULL == found_node) {
 839                 return _MALI_OSK_ERR_FAULT;
 840         }
 841
 842         found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx);
 843
 844         if (!IS_ERR(found_page)) {
 845                 lock_page(found_page);
 846                 dma_addr = dma_map_page(&mali_platform_device->dev, found_page,
 847                                         0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
 848                 dma_unmap_page(&mali_platform_device->dev, dma_addr,
 849                                _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
 850
 851                 *pagep = found_page;
 852         } else {
 853                 return _MALI_OSK_ERR_NOMEM;
 854         }
 855
 856         return _MALI_OSK_ERR_OK;
 857 }
 858
 859 int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep)
 860 {
 861         struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL;
 862         mali_mem_cow *cow = NULL;
 863         u32 i = 0;
 864
 865         MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
 866         MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED));
 867         MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags));
 868         MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend));
 869
 870         cow = &mem_bkend->cow_mem;
 871         list_for_each_entry(m_page, &cow->pages, list) {
 872                 if (i == offset) {
 873                         found_node = m_page;
 874                         break;
 875                 }
 876                 i++;
 877         }
 878
 879         if (NULL == found_node) {
 880                 return _MALI_OSK_ERR_FAULT;
 881         }
 882
 883         new_node = _mali_mem_swap_page_node_allocate();
 884
 885         if (NULL == new_node) {
 886                 return _MALI_OSK_ERR_FAULT;
 887         }
 888
 889         new_node->swap_it->idx = mali_mem_swap_idx_alloc();
 890
 891         if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) {
 892                 MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n"));
 893                 kfree(new_node->swap_it);
 894                 kfree(new_node);
 895                 return _MALI_OSK_ERR_FAULT;
 896         }
 897
 898         if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) {
 899                 _mali_mem_swap_page_node_free(new_node);
 900                 return _MALI_OSK_ERR_FAULT;
 901         }
 902
 903         /* swap in found node for copy in kernel. */
 904         if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) {
 905                 mali_mem_swap_out_page_node(new_node);
 906                 _mali_mem_swap_page_node_free(new_node);
 907                 return _MALI_OSK_ERR_FAULT;
 908         }
 909
 910         _mali_mem_cow_copy_page(found_node, new_node);
 911
 912         list_replace(&found_node->list, &new_node->list);
 913
 914         if (1 != _mali_page_node_get_ref_count(found_node)) {
 915                 atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages);
 916                 if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) {
 917                         mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
 918                 }
 919                 mem_bkend->cow_mem.change_pages_nr++;
 920         }
 921
 922         mali_mem_swap_out_page_node(found_node);
 923         _mali_mem_swap_page_node_free(found_node);
 924
 925         /* When swap in the new page node, we have called dma_map_page for this page.\n */
 926         dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr,
 927                        _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
 928
 929         lock_page(new_node->swap_it->page);
 930
 931         *pagep = new_node->swap_it->page;
 932
 933         return _MALI_OSK_ERR_OK;
 934 }
 935
 936 #ifdef MALI_MEM_SWAP_TRACKING
 937 void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size)
 938 {
 939         *swap_pool_size = mem_backend_swapped_pool_size;
 940         *unlock_size =  mem_backend_swapped_unlock_size;
 941 }
 942 #endif