drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c

   1 /*
   2  * Copyright (C) 2011-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_osk.h"
  13 #include "mali_ukk.h"
  14 #include "mali_uk_types.h"
  15 #include "mali_mmu_page_directory.h"
  16 #include "mali_memory.h"
  17 #include "mali_l2_cache.h"
  18
  19 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);
  20
  21 u32 mali_allocate_empty_page(mali_io_address *virt_addr)
  22 {
  23         _mali_osk_errcode_t err;
  24         mali_io_address mapping;
  25         mali_dma_addr address;
  26
  27         if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
  28                 /* Allocation failed */
  29                 MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
  30                 return 0;
  31         }
  32
  33         MALI_DEBUG_ASSERT_POINTER(mapping);
  34
  35         err = fill_page(mapping, 0);
  36         if (_MALI_OSK_ERR_OK != err) {
  37                 mali_mmu_release_table_page(address, mapping);
  38                 MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
  39                 return 0;
  40         }
  41
  42         *virt_addr = mapping;
  43         return address;
  44 }
  45
  46 void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
  47 {
  48         if (MALI_INVALID_PAGE != address) {
  49                 mali_mmu_release_table_page(address, virt_addr);
  50         }
  51 }
  52
  53 _mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
  54                 mali_io_address *page_directory_mapping,
  55                 mali_dma_addr *page_table, mali_io_address *page_table_mapping,
  56                 mali_dma_addr *data_page, mali_io_address *data_page_mapping)
  57 {
  58         _mali_osk_errcode_t err;
  59
  60         err = mali_mmu_get_table_page(data_page, data_page_mapping);
  61         if (_MALI_OSK_ERR_OK == err) {
  62                 err = mali_mmu_get_table_page(page_table, page_table_mapping);
  63                 if (_MALI_OSK_ERR_OK == err) {
  64                         err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
  65                         if (_MALI_OSK_ERR_OK == err) {
  66                                 fill_page(*data_page_mapping, 0);
  67                                 fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT);
  68                                 fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT);
  69                                 MALI_SUCCESS;
  70                         }
  71                         mali_mmu_release_table_page(*page_table, *page_table_mapping);
  72                         *page_table = MALI_INVALID_PAGE;
  73                 }
  74                 mali_mmu_release_table_page(*data_page, *data_page_mapping);
  75                 *data_page = MALI_INVALID_PAGE;
  76         }
  77         return err;
  78 }
  79
  80 void mali_destroy_fault_flush_pages(
  81         mali_dma_addr *page_directory, mali_io_address *page_directory_mapping,
  82         mali_dma_addr *page_table, mali_io_address *page_table_mapping,
  83         mali_dma_addr *data_page, mali_io_address *data_page_mapping)
  84 {
  85         if (MALI_INVALID_PAGE != *page_directory) {
  86                 mali_mmu_release_table_page(*page_directory, *page_directory_mapping);
  87                 *page_directory = MALI_INVALID_PAGE;
  88                 *page_directory_mapping = NULL;
  89         }
  90
  91         if (MALI_INVALID_PAGE != *page_table) {
  92                 mali_mmu_release_table_page(*page_table, *page_table_mapping);
  93                 *page_table = MALI_INVALID_PAGE;
  94                 *page_table_mapping = NULL;
  95         }
  96
  97         if (MALI_INVALID_PAGE != *data_page) {
  98                 mali_mmu_release_table_page(*data_page, *data_page_mapping);
  99                 *data_page = MALI_INVALID_PAGE;
 100                 *data_page_mapping = NULL;
 101         }
 102 }
 103
 104 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
 105 {
 106         int i;
 107         MALI_DEBUG_ASSERT_POINTER(mapping);
 108
 109         for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
 110                 _mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
 111         }
 112         _mali_osk_mem_barrier();
 113         MALI_SUCCESS;
 114 }
 115
 116 _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 117 {
 118         const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 119         const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 120         _mali_osk_errcode_t err;
 121         mali_io_address pde_mapping;
 122         mali_dma_addr pde_phys;
 123         int i, page_count;
 124         u32 start_address;
 125         if (last_pde < first_pde)
 126                 return _MALI_OSK_ERR_INVALID_ARGS;
 127
 128         for (i = first_pde; i <= last_pde; i++) {
 129                 if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 130                                                  i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
 131                         /* Page table not present */
 132                         MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 133                         MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);
 134
 135                         err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
 136                         if (_MALI_OSK_ERR_OK != err) {
 137                                 MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
 138                                 return err;
 139                         }
 140                         pagedir->page_entries_mapped[i] = pde_mapping;
 141
 142                         /* Update PDE, mark as present */
 143                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
 144                                                         pde_phys | MALI_MMU_FLAGS_PRESENT);
 145
 146                         MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 147                 }
 148
 149                 if (first_pde == last_pde) {
 150                         pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE;
 151                 } else if (i == first_pde) {
 152                         start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
 153                         page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE;
 154                         pagedir->page_entries_usage_count[i] += page_count;
 155                 } else if (i == last_pde) {
 156                         start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
 157                         page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE;
 158                         pagedir->page_entries_usage_count[i] += page_count;
 159                 } else {
 160                         pagedir->page_entries_usage_count[i] = 1024;
 161                 }
 162         }
 163         _mali_osk_write_mem_barrier();
 164
 165         return _MALI_OSK_ERR_OK;
 166 }
 167
 168 MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
 169 {
 170         int i;
 171         const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
 172         const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);
 173
 174         for (i = first_pte; i <= last_pte; i++) {
 175                 _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
 176         }
 177 }
 178
 179 static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
 180 {
 181         return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 182                                        index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 183 }
 184
 185
 186 _mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 187 {
 188         const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 189         const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 190         u32 left = size;
 191         int i;
 192         mali_bool pd_changed = MALI_FALSE;
 193         u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
 194         u32 num_pages_inv = 0;
 195         mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */
 196
 197         /* For all page directory entries in range. */
 198         for (i = first_pde; i <= last_pde; i++) {
 199                 u32 size_in_pde, offset;
 200
 201                 MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
 202                 MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);
 203
 204                 /* Offset into page table, 0 if mali_address is 4MiB aligned */
 205                 offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
 206                 if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
 207                         size_in_pde = left;
 208                 } else {
 209                         size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
 210                 }
 211
 212                 pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE;
 213
 214                 /* If entire page table is unused, free it */
 215                 if (0 == pagedir->page_entries_usage_count[i]) {
 216                         u32 page_phys;
 217                         void *page_virt;
 218                         MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
 219                         /* last reference removed, no need to zero out each PTE  */
 220
 221                         page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
 222                         page_virt = pagedir->page_entries_mapped[i];
 223                         pagedir->page_entries_mapped[i] = NULL;
 224                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
 225
 226                         mali_mmu_release_table_page(page_phys, page_virt);
 227                         pd_changed = MALI_TRUE;
 228                 } else {
 229                         MALI_DEBUG_ASSERT(num_pages_inv < 2);
 230                         if (num_pages_inv < 2) {
 231                                 pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
 232                                 num_pages_inv++;
 233                         } else {
 234                                 invalidate_all = MALI_TRUE;
 235                         }
 236
 237                         /* If part of the page table is still in use, zero the relevant PTEs */
 238                         mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
 239                 }
 240
 241                 left -= size_in_pde;
 242                 mali_address += size_in_pde;
 243         }
 244         _mali_osk_write_mem_barrier();
 245
 246         /* L2 pages invalidation */
 247         if (MALI_TRUE == pd_changed) {
 248                 MALI_DEBUG_ASSERT(num_pages_inv < 3);
 249                 if (num_pages_inv < 3) {
 250                         pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
 251                         num_pages_inv++;
 252                 } else {
 253                         invalidate_all = MALI_TRUE;
 254                 }
 255         }
 256
 257         if (invalidate_all) {
 258                 mali_l2_cache_invalidate_all();
 259         } else {
 260                 mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
 261         }
 262
 263         MALI_SUCCESS;
 264 }
 265
 266 struct mali_page_directory *mali_mmu_pagedir_alloc(void)
 267 {
 268         struct mali_page_directory *pagedir;
 269         _mali_osk_errcode_t err;
 270         mali_dma_addr phys;
 271
 272         pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
 273         if (NULL == pagedir) {
 274                 return NULL;
 275         }
 276
 277         err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
 278         if (_MALI_OSK_ERR_OK != err) {
 279                 _mali_osk_free(pagedir);
 280                 return NULL;
 281         }
 282
 283         pagedir->page_directory = (u32)phys;
 284
 285         /* Zero page directory */
 286         fill_page(pagedir->page_directory_mapped, 0);
 287
 288         return pagedir;
 289 }
 290
 291 void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
 292 {
 293         const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
 294         int i;
 295
 296         /* Free referenced page tables and zero PDEs. */
 297         for (i = 0; i < num_page_table_entries; i++) {
 298                 if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
 299                                 pagedir->page_directory_mapped,
 300                                 sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
 301                         mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 302                                              i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
 303                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
 304                         mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
 305                 }
 306         }
 307         _mali_osk_write_mem_barrier();
 308
 309         /* Free the page directory page. */
 310         mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);
 311
 312         _mali_osk_free(pagedir);
 313 }
 314
 315
 316 void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
 317                              mali_dma_addr phys_address, u32 size, u32 permission_bits)
 318 {
 319         u32 end_address = mali_address + size;
 320         u32 mali_phys = (u32)phys_address;
 321
 322         /* Map physical pages into MMU page tables */
 323         for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
 324                 MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
 325                 _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
 326                                                 MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
 327                                                 mali_phys | permission_bits);
 328         }
 329 }
 330
 331 void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
 332 {
 333 #if defined(DEBUG)
 334         u32 pde_index, pte_index;
 335         u32 pde, pte;
 336
 337         pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
 338         pte_index = MALI_MMU_PTE_ENTRY(fault_addr);
 339
 340
 341         pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 342                                      pde_index * sizeof(u32));
 343
 344
 345         if (pde & MALI_MMU_FLAGS_PRESENT) {
 346                 u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);
 347
 348                 pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
 349                                              pte_index * sizeof(u32));
 350
 351                 MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
 352                                      "\t\tPTE: %08x, page %08x is %s\n",
 353                                      fault_addr, pte_addr, pte,
 354                                      MALI_MMU_ENTRY_ADDRESS(pte),
 355                                      pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
 356         } else {
 357                 MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
 358                                      fault_addr, pde));
 359         }
 360 #else
 361         MALI_IGNORE(pagedir);
 362         MALI_IGNORE(fault_addr);
 363 #endif
 364 }
 365
 366 /* For instrumented */
 367 struct dump_info {
 368         u32 buffer_left;
 369         u32 register_writes_size;
 370         u32 page_table_dump_size;
 371         u32 *buffer;
 372 };
 373
 374 static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info)
 375 {
 376         if (NULL != info) {
 377                 info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */
 378
 379                 if (NULL != info->buffer) {
 380                         /* check that we have enough space */
 381                         if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
 382
 383                         *info->buffer = where;
 384                         info->buffer++;
 385
 386                         *info->buffer = what;
 387                         info->buffer++;
 388
 389                         info->buffer_left -= sizeof(u32) * 2;
 390                 }
 391         }
 392
 393         MALI_SUCCESS;
 394 }
 395
 396 static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
 397 {
 398         if (NULL != info) {
 399                 /* 4096 for the page and 4 bytes for the address */
 400                 const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
 401                 const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
 402                 const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;
 403
 404                 info->page_table_dump_size += dump_size_in_bytes;
 405
 406                 if (NULL != info->buffer) {
 407                         if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
 408
 409                         *info->buffer = phys_addr;
 410                         info->buffer++;
 411
 412                         _mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
 413                         info->buffer += page_size_in_elements;
 414
 415                         info->buffer_left -= dump_size_in_bytes;
 416                 }
 417         }
 418
 419         MALI_SUCCESS;
 420 }
 421
 422 static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info)
 423 {
 424         MALI_DEBUG_ASSERT_POINTER(pagedir);
 425         MALI_DEBUG_ASSERT_POINTER(info);
 426
 427         if (NULL != pagedir->page_directory_mapped) {
 428                 int i;
 429
 430                 MALI_CHECK_NO_ERROR(
 431                         mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
 432                 );
 433
 434                 for (i = 0; i < 1024; i++) {
 435                         if (NULL != pagedir->page_entries_mapped[i]) {
 436                                 MALI_CHECK_NO_ERROR(
 437                                         mali_mmu_dump_page(pagedir->page_entries_mapped[i],
 438                                                            _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 439                                                                            i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
 440                                 );
 441                         }
 442                 }
 443         }
 444
 445         MALI_SUCCESS;
 446 }
 447
 448 static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info)
 449 {
 450         MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
 451                                      "set the page directory address", info));
 452         MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
 453         MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
 454         MALI_SUCCESS;
 455 }
 456
 457 _mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
 458 {
 459         struct dump_info info = { 0, 0, 0, NULL };
 460         struct mali_session_data *session_data;
 461
 462         session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
 463         MALI_DEBUG_ASSERT_POINTER(session_data);
 464         MALI_DEBUG_ASSERT_POINTER(args);
 465
 466         MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
 467         MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
 468         args->size = info.register_writes_size + info.page_table_dump_size;
 469         MALI_SUCCESS;
 470 }
 471
 472 _mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
 473 {
 474         struct dump_info info = { 0, 0, 0, NULL };
 475         struct mali_session_data *session_data;
 476
 477         MALI_DEBUG_ASSERT_POINTER(args);
 478
 479         session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
 480         MALI_DEBUG_ASSERT_POINTER(session_data);
 481
 482         info.buffer_left = args->size;
 483         info.buffer = (u32 *)(uintptr_t)args->buffer;
 484
 485         args->register_writes = (uintptr_t)info.buffer;
 486         MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
 487
 488         args->page_table_dump = (uintptr_t)info.buffer;
 489         MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
 490
 491         args->register_writes_size = info.register_writes_size;
 492         args->page_table_dump_size = info.page_table_dump_size;
 493
 494         MALI_SUCCESS;
 495 }