drivers/gpu/arm/midgard_for_linux/mali_kbase_mmu.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_mmu.c
  22  * Base kernel MMU management.
  23  */
  24
  25 /* #define DEBUG    1 */
  26 #include <linux/kernel.h>
  27 #include <linux/dma-mapping.h>
  28 #include <mali_kbase.h>
  29 #include <mali_midg_regmap.h>
  30 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  31 #include <mali_kbase_gator.h>
  32 #endif
  33 #include <mali_kbase_tlstream.h>
  34 #include <mali_kbase_instr_defs.h>
  35 #include <mali_kbase_debug.h>
  36
  37 #define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  38
  39 #include <mali_kbase_defs.h>
  40 #include <mali_kbase_hw.h>
  41 #include <mali_kbase_mmu_hw.h>
  42 #include <mali_kbase_hwaccess_jm.h>
  43 #include <mali_kbase_time.h>
  44
  45 #define KBASE_MMU_PAGE_ENTRIES 512
  46
  47 /**
  48  * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches.
  49  * @kctx: The KBase context.
  50  * @vpfn: The virtual page frame number to start the flush on.
  51  * @nr: The number of pages to flush.
  52  * @sync: Set if the operation should be synchronous or not.
  53  *
  54  * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs.
  55  *
  56  * If sync is not set then transactions still in flight when the flush is issued
  57  * may use the old page tables and the data they write will not be written out
  58  * to memory, this function returns after the flush has been issued but
  59  * before all accesses which might effect the flushed region have completed.
  60  *
  61  * If sync is set then accesses in the flushed region will be drained
  62  * before data is flush and invalidated through L1, L2 and into memory,
  63  * after which point this function will return.
  64  */
  65 static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
  66                 u64 vpfn, size_t nr, bool sync);
  67
  68 /**
  69  * kbase_mmu_sync_pgd - sync page directory to memory
  70  * @kbdev:      Device pointer.
  71  * @handle:     Address of DMA region.
  72  * @size:       Size of the region to sync.
  73  *
  74  * This should be called after each page directory update.
  75  */
  76
  77 static void kbase_mmu_sync_pgd(struct kbase_device *kbdev,
  78                 dma_addr_t handle, size_t size)
  79 {
  80         /* If page table is not coherent then ensure the gpu can read
  81          * the pages from memory
  82          */
  83         if (kbdev->system_coherency != COHERENCY_ACE)
  84                 dma_sync_single_for_device(kbdev->dev, handle, size,
  85                                 DMA_TO_DEVICE);
  86 }
  87
  88 /*
  89  * Definitions:
  90  * - PGD: Page Directory.
  91  * - PTE: Page Table Entry. A 64bit value pointing to the next
  92  *        level of translation
  93  * - ATE: Address Transation Entry. A 64bit value pointing to
  94  *        a 4kB physical page.
  95  */
  96
  97 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
  98                 struct kbase_as *as, const char *reason_str);
  99
 100
 101 static size_t make_multiple(size_t minimum, size_t multiple)
 102 {
 103         size_t remainder = minimum % multiple;
 104
 105         if (remainder == 0)
 106                 return minimum;
 107
 108         return minimum + multiple - remainder;
 109 }
 110
 111 void page_fault_worker(struct work_struct *data)
 112 {
 113         u64 fault_pfn;
 114         u32 fault_status;
 115         size_t new_pages;
 116         size_t fault_rel_pfn;
 117         struct kbase_as *faulting_as;
 118         int as_no;
 119         struct kbase_context *kctx;
 120         struct kbase_device *kbdev;
 121         struct kbase_va_region *region;
 122         int err;
 123         bool grown = false;
 124
 125         faulting_as = container_of(data, struct kbase_as, work_pagefault);
 126         fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
 127         as_no = faulting_as->number;
 128
 129         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
 130
 131         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
 132          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
 133          */
 134         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
 135         if (WARN_ON(!kctx)) {
 136                 atomic_dec(&kbdev->faults_pending);
 137                 return;
 138         }
 139
 140         KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
 141
 142         fault_status = faulting_as->fault_status;
 143         switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
 144
 145         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
 146                 /* need to check against the region to handle this one */
 147                 break;
 148
 149         case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
 150                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 151                                 "Permission failure");
 152                 goto fault_done;
 153
 154         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
 155                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 156                                 "Tranlation table bus fault");
 157                 goto fault_done;
 158
 159         case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
 160                 /* nothing to do, but we don't expect this fault currently */
 161                 dev_warn(kbdev->dev, "Access flag unexpectedly set");
 162                 goto fault_done;
 163
 164 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
 165         case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT:
 166
 167                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 168                                         "Address size fault");
 169                 goto fault_done;
 170
 171         case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT:
 172                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 173                                         "Memory attributes fault");
 174                 goto fault_done;
 175 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
 176
 177         default:
 178                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 179                                 "Unknown fault code");
 180                 goto fault_done;
 181         }
 182
 183         /* so we have a translation fault, let's see if it is for growable
 184          * memory */
 185         kbase_gpu_vm_lock(kctx);
 186
 187         region = kbase_region_tracker_find_region_enclosing_address(kctx,
 188                         faulting_as->fault_addr);
 189         if (!region || region->flags & KBASE_REG_FREE) {
 190                 kbase_gpu_vm_unlock(kctx);
 191                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 192                                 "Memory is not mapped on the GPU");
 193                 goto fault_done;
 194         }
 195
 196         if ((region->flags & GROWABLE_FLAGS_REQUIRED)
 197                         != GROWABLE_FLAGS_REQUIRED) {
 198                 kbase_gpu_vm_unlock(kctx);
 199                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 200                                 "Memory is not growable");
 201                 goto fault_done;
 202         }
 203
 204         if ((region->flags & KBASE_REG_DONT_NEED)) {
 205                 kbase_gpu_vm_unlock(kctx);
 206                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 207                                 "Don't need memory can't be grown");
 208                 goto fault_done;
 209         }
 210
 211         /* find the size we need to grow it by */
 212         /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
 213          * validating the fault_adress to be within a size_t from the start_pfn */
 214         fault_rel_pfn = fault_pfn - region->start_pfn;
 215
 216         if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
 217                 dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
 218                                 faulting_as->fault_addr, region->start_pfn,
 219                                 region->start_pfn +
 220                                 kbase_reg_current_backed_size(region));
 221
 222                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 223                                 KBASE_MMU_FAULT_TYPE_PAGE);
 224                 /* [1] in case another page fault occurred while we were
 225                  * handling the (duplicate) page fault we need to ensure we
 226                  * don't loose the other page fault as result of us clearing
 227                  * the MMU IRQ. Therefore, after we clear the MMU IRQ we send
 228                  * an UNLOCK command that will retry any stalled memory
 229                  * transaction (which should cause the other page fault to be
 230                  * raised again).
 231                  */
 232                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 233                                 AS_COMMAND_UNLOCK, 1);
 234                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 235                                 KBASE_MMU_FAULT_TYPE_PAGE);
 236                 kbase_gpu_vm_unlock(kctx);
 237
 238                 goto fault_done;
 239         }
 240
 241         new_pages = make_multiple(fault_rel_pfn -
 242                         kbase_reg_current_backed_size(region) + 1,
 243                         region->extent);
 244
 245         /* cap to max vsize */
 246         if (new_pages + kbase_reg_current_backed_size(region) >
 247                         region->nr_pages)
 248                 new_pages = region->nr_pages -
 249                                 kbase_reg_current_backed_size(region);
 250
 251         if (0 == new_pages) {
 252                 /* Duplicate of a fault we've already handled, nothing to do */
 253                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 254                                 KBASE_MMU_FAULT_TYPE_PAGE);
 255                 /* See comment [1] about UNLOCK usage */
 256                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 257                                 AS_COMMAND_UNLOCK, 1);
 258                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 259                                 KBASE_MMU_FAULT_TYPE_PAGE);
 260                 kbase_gpu_vm_unlock(kctx);
 261                 goto fault_done;
 262         }
 263
 264         if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
 265                 if (region->gpu_alloc != region->cpu_alloc) {
 266                         if (kbase_alloc_phy_pages_helper(
 267                                         region->cpu_alloc, new_pages) == 0) {
 268                                 grown = true;
 269                         } else {
 270                                 kbase_free_phy_pages_helper(region->gpu_alloc,
 271                                                 new_pages);
 272                         }
 273                 } else {
 274                         grown = true;
 275                 }
 276         }
 277
 278
 279         if (grown) {
 280                 u64 pfn_offset;
 281                 u32 op;
 282
 283                 /* alloc success */
 284                 KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
 285
 286                 /* set up the new pages */
 287                 pfn_offset = kbase_reg_current_backed_size(region) - new_pages;
 288                 /*
 289                  * Note:
 290                  * Issuing an MMU operation will unlock the MMU and cause the
 291                  * translation to be replayed. If the page insertion fails then
 292                  * rather then trying to continue the context should be killed
 293                  * so the no_flush version of insert_pages is used which allows
 294                  * us to unlock the MMU as we see fit.
 295                  */
 296                 err = kbase_mmu_insert_pages_no_flush(kctx,
 297                                 region->start_pfn + pfn_offset,
 298                                 &kbase_get_gpu_phy_pages(region)[pfn_offset],
 299                                 new_pages, region->flags);
 300                 if (err) {
 301                         kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
 302                         if (region->gpu_alloc != region->cpu_alloc)
 303                                 kbase_free_phy_pages_helper(region->cpu_alloc,
 304                                                 new_pages);
 305                         kbase_gpu_vm_unlock(kctx);
 306                         /* The locked VA region will be unlocked and the cache invalidated in here */
 307                         kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 308                                         "Page table update failure");
 309                         goto fault_done;
 310                 }
 311 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 312                 kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
 313 #endif
 314                 kbase_tlstream_aux_pagefault(kctx->id, (u64)new_pages);
 315
 316                 /* AS transaction begin */
 317                 mutex_lock(&faulting_as->transaction_mutex);
 318
 319                 /* flush L2 and unlock the VA (resumes the MMU) */
 320                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
 321                         op = AS_COMMAND_FLUSH;
 322                 else
 323                         op = AS_COMMAND_FLUSH_PT;
 324
 325                 /* clear MMU interrupt - this needs to be done after updating
 326                  * the page tables but before issuing a FLUSH command. The
 327                  * FLUSH cmd has a side effect that it restarts stalled memory
 328                  * transactions in other address spaces which may cause
 329                  * another fault to occur. If we didn't clear the interrupt at
 330                  * this stage a new IRQ might not be raised when the GPU finds
 331                  * a MMU IRQ is already pending.
 332                  */
 333                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 334                                          KBASE_MMU_FAULT_TYPE_PAGE);
 335
 336                 kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
 337                                           faulting_as->fault_addr >> PAGE_SHIFT,
 338                                           new_pages,
 339                                           op, 1);
 340
 341                 mutex_unlock(&faulting_as->transaction_mutex);
 342                 /* AS transaction end */
 343
 344                 /* reenable this in the mask */
 345                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 346                                          KBASE_MMU_FAULT_TYPE_PAGE);
 347                 kbase_gpu_vm_unlock(kctx);
 348         } else {
 349                 /* failed to extend, handle as a normal PF */
 350                 kbase_gpu_vm_unlock(kctx);
 351                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 352                                 "Page allocation failure");
 353         }
 354
 355 fault_done:
 356         /*
 357          * By this point, the fault was handled in some way,
 358          * so release the ctx refcount
 359          */
 360         kbasep_js_runpool_release_ctx(kbdev, kctx);
 361
 362         atomic_dec(&kbdev->faults_pending);
 363 }
 364
 365 phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx)
 366 {
 367         u64 *page;
 368         int i;
 369         struct page *p;
 370         int new_page_count __maybe_unused;
 371
 372         KBASE_DEBUG_ASSERT(NULL != kctx);
 373         new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages);
 374         kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages);
 375
 376         p = kbase_mem_pool_alloc(&kctx->mem_pool);
 377         if (!p)
 378                 goto sub_pages;
 379
 380         kbase_tlstream_aux_pagesalloc(
 381                         (u32)kctx->id,
 382                         (u64)new_page_count);
 383
 384         page = kmap(p);
 385         if (NULL == page)
 386                 goto alloc_free;
 387
 388         kbase_process_page_usage_inc(kctx, 1);
 389
 390         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++)
 391                 kctx->kbdev->mmu_mode->entry_invalidate(&page[i]);
 392
 393         kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 394
 395         kunmap(p);
 396         return page_to_phys(p);
 397
 398 alloc_free:
 399         kbase_mem_pool_free(&kctx->mem_pool, p, false);
 400 sub_pages:
 401         kbase_atomic_sub_pages(1, &kctx->used_pages);
 402         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
 403
 404         return 0;
 405 }
 406
 407 KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd);
 408
 409 /* Given PGD PFN for level N, return PGD PFN for level N+1 */
 410 static phys_addr_t mmu_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 411 {
 412         u64 *page;
 413         phys_addr_t target_pgd;
 414         struct page *p;
 415
 416         KBASE_DEBUG_ASSERT(pgd);
 417         KBASE_DEBUG_ASSERT(NULL != kctx);
 418
 419         lockdep_assert_held(&kctx->mmu_lock);
 420
 421         /*
 422          * Architecture spec defines level-0 as being the top-most.
 423          * This is a bit unfortunate here, but we keep the same convention.
 424          */
 425         vpfn >>= (3 - level) * 9;
 426         vpfn &= 0x1FF;
 427
 428         p = pfn_to_page(PFN_DOWN(pgd));
 429         page = kmap(p);
 430         if (NULL == page) {
 431                 dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n");
 432                 return 0;
 433         }
 434
 435         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 436
 437         if (!target_pgd) {
 438                 target_pgd = kbase_mmu_alloc_pgd(kctx);
 439                 if (!target_pgd) {
 440                         dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n");
 441                         kunmap(p);
 442                         return 0;
 443                 }
 444
 445                 kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd);
 446
 447                 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 448                 /* Rely on the caller to update the address space flags. */
 449         }
 450
 451         kunmap(p);
 452         return target_pgd;
 453 }
 454
 455 static phys_addr_t mmu_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 456 {
 457         phys_addr_t pgd;
 458         int l;
 459
 460         lockdep_assert_held(&kctx->mmu_lock);
 461
 462         pgd = kctx->pgd;
 463         for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
 464                 pgd = mmu_get_next_pgd(kctx, pgd, vpfn, l);
 465                 /* Handle failure condition */
 466                 if (!pgd) {
 467                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n");
 468                         return 0;
 469                 }
 470         }
 471
 472         return pgd;
 473 }
 474
 475 static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 476 {
 477         u64 *page;
 478         phys_addr_t target_pgd;
 479
 480         KBASE_DEBUG_ASSERT(pgd);
 481         KBASE_DEBUG_ASSERT(NULL != kctx);
 482
 483         lockdep_assert_held(&kctx->mmu_lock);
 484
 485         /*
 486          * Architecture spec defines level-0 as being the top-most.
 487          * This is a bit unfortunate here, but we keep the same convention.
 488          */
 489         vpfn >>= (3 - level) * 9;
 490         vpfn &= 0x1FF;
 491
 492         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 493         /* kmap_atomic should NEVER fail */
 494         KBASE_DEBUG_ASSERT(NULL != page);
 495
 496         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 497         /* As we are recovering from what has already been set up, we should have a target_pgd */
 498         KBASE_DEBUG_ASSERT(0 != target_pgd);
 499         kunmap_atomic(page);
 500         return target_pgd;
 501 }
 502
 503 static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 504 {
 505         phys_addr_t pgd;
 506         int l;
 507
 508         lockdep_assert_held(&kctx->mmu_lock);
 509
 510         pgd = kctx->pgd;
 511
 512         for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
 513                 pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l);
 514                 /* Should never fail */
 515                 KBASE_DEBUG_ASSERT(0 != pgd);
 516         }
 517
 518         return pgd;
 519 }
 520
 521 static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn,
 522                                               size_t nr)
 523 {
 524         phys_addr_t pgd;
 525         u64 *pgd_page;
 526         struct kbase_mmu_mode const *mmu_mode;
 527
 528         KBASE_DEBUG_ASSERT(NULL != kctx);
 529         KBASE_DEBUG_ASSERT(0 != vpfn);
 530         /* 64-bit address range is the max */
 531         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 532
 533         lockdep_assert_held(&kctx->mmu_lock);
 534
 535         mmu_mode = kctx->kbdev->mmu_mode;
 536
 537         while (nr) {
 538                 unsigned int i;
 539                 unsigned int index = vpfn & 0x1FF;
 540                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 541                 struct page *p;
 542
 543                 if (count > nr)
 544                         count = nr;
 545
 546                 pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn);
 547                 KBASE_DEBUG_ASSERT(0 != pgd);
 548
 549                 p = pfn_to_page(PFN_DOWN(pgd));
 550
 551                 pgd_page = kmap_atomic(p);
 552                 KBASE_DEBUG_ASSERT(NULL != pgd_page);
 553
 554                 /* Invalidate the entries we added */
 555                 for (i = 0; i < count; i++)
 556                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
 557
 558                 vpfn += count;
 559                 nr -= count;
 560
 561                 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 562
 563                 kunmap_atomic(pgd_page);
 564         }
 565 }
 566
 567 /*
 568  * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn'
 569  */
 570 int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
 571                                         phys_addr_t phys, size_t nr,
 572                                         unsigned long flags)
 573 {
 574         phys_addr_t pgd;
 575         u64 *pgd_page;
 576         /* In case the insert_single_page only partially completes we need to be
 577          * able to recover */
 578         bool recover_required = false;
 579         u64 recover_vpfn = vpfn;
 580         size_t recover_count = 0;
 581         size_t remain = nr;
 582         int err;
 583
 584         KBASE_DEBUG_ASSERT(NULL != kctx);
 585         KBASE_DEBUG_ASSERT(0 != vpfn);
 586         /* 64-bit address range is the max */
 587         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 588
 589         /* Early out if there is nothing to do */
 590         if (nr == 0)
 591                 return 0;
 592
 593         mutex_lock(&kctx->mmu_lock);
 594
 595         while (remain) {
 596                 unsigned int i;
 597                 unsigned int index = vpfn & 0x1FF;
 598                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 599                 struct page *p;
 600
 601                 if (count > remain)
 602                         count = remain;
 603
 604                 /*
 605                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 606                  * suboptimal. We don't have to re-parse the whole tree
 607                  * each time (just cache the l0-l2 sequence).
 608                  * On the other hand, it's only a gain when we map more than
 609                  * 256 pages at once (on average). Do we really care?
 610                  */
 611                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 612                 if (!pgd) {
 613                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 614                         if (recover_required) {
 615                                 /* Invalidate the pages we have partially
 616                                  * completed */
 617                                 mmu_insert_pages_failure_recovery(kctx,
 618                                                                   recover_vpfn,
 619                                                                   recover_count);
 620                         }
 621                         err = -EINVAL;
 622                         goto fail_unlock;
 623                 }
 624
 625                 p = pfn_to_page(PFN_DOWN(pgd));
 626                 pgd_page = kmap(p);
 627                 if (!pgd_page) {
 628                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 629                         if (recover_required) {
 630                                 /* Invalidate the pages we have partially
 631                                  * completed */
 632                                 mmu_insert_pages_failure_recovery(kctx,
 633                                                                   recover_vpfn,
 634                                                                   recover_count);
 635                         }
 636                         err = -ENOMEM;
 637                         goto fail_unlock;
 638                 }
 639
 640                 for (i = 0; i < count; i++) {
 641                         unsigned int ofs = index + i;
 642
 643                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 644                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 645                                         phys, flags);
 646                 }
 647
 648                 vpfn += count;
 649                 remain -= count;
 650
 651                 kbase_mmu_sync_pgd(kctx->kbdev,
 652                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 653                                 count * sizeof(u64));
 654
 655                 kunmap(p);
 656                 /* We have started modifying the page table.
 657                  * If further pages need inserting and fail we need to undo what
 658                  * has already taken place */
 659                 recover_required = true;
 660                 recover_count += count;
 661         }
 662         mutex_unlock(&kctx->mmu_lock);
 663         kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
 664         return 0;
 665
 666 fail_unlock:
 667         mutex_unlock(&kctx->mmu_lock);
 668         kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
 669         return err;
 670 }
 671
 672 int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn,
 673                                   phys_addr_t *phys, size_t nr,
 674                                   unsigned long flags)
 675 {
 676         phys_addr_t pgd;
 677         u64 *pgd_page;
 678         /* In case the insert_pages only partially completes we need to be able
 679          * to recover */
 680         bool recover_required = false;
 681         u64 recover_vpfn = vpfn;
 682         size_t recover_count = 0;
 683         size_t remain = nr;
 684         int err;
 685
 686         KBASE_DEBUG_ASSERT(NULL != kctx);
 687         KBASE_DEBUG_ASSERT(0 != vpfn);
 688         /* 64-bit address range is the max */
 689         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 690
 691         /* Early out if there is nothing to do */
 692         if (nr == 0)
 693                 return 0;
 694
 695         mutex_lock(&kctx->mmu_lock);
 696
 697         while (remain) {
 698                 unsigned int i;
 699                 unsigned int index = vpfn & 0x1FF;
 700                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 701                 struct page *p;
 702
 703                 if (count > remain)
 704                         count = remain;
 705
 706                 /*
 707                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 708                  * suboptimal. We don't have to re-parse the whole tree
 709                  * each time (just cache the l0-l2 sequence).
 710                  * On the other hand, it's only a gain when we map more than
 711                  * 256 pages at once (on average). Do we really care?
 712                  */
 713                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 714                 if (!pgd) {
 715                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 716                         if (recover_required) {
 717                                 /* Invalidate the pages we have partially
 718                                  * completed */
 719                                 mmu_insert_pages_failure_recovery(kctx,
 720                                                                   recover_vpfn,
 721                                                                   recover_count);
 722                         }
 723                         err = -EINVAL;
 724                         goto fail_unlock;
 725                 }
 726
 727                 p = pfn_to_page(PFN_DOWN(pgd));
 728                 pgd_page = kmap(p);
 729                 if (!pgd_page) {
 730                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 731                         if (recover_required) {
 732                                 /* Invalidate the pages we have partially
 733                                  * completed */
 734                                 mmu_insert_pages_failure_recovery(kctx,
 735                                                                   recover_vpfn,
 736                                                                   recover_count);
 737                         }
 738                         err = -ENOMEM;
 739                         goto fail_unlock;
 740                 }
 741
 742                 for (i = 0; i < count; i++) {
 743                         unsigned int ofs = index + i;
 744
 745                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 746                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 747                                         phys[i], flags);
 748                 }
 749
 750                 phys += count;
 751                 vpfn += count;
 752                 remain -= count;
 753
 754                 kbase_mmu_sync_pgd(kctx->kbdev,
 755                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 756                                 count * sizeof(u64));
 757
 758                 kunmap(p);
 759                 /* We have started modifying the page table. If further pages
 760                  * need inserting and fail we need to undo what has already
 761                  * taken place */
 762                 recover_required = true;
 763                 recover_count += count;
 764         }
 765
 766         mutex_unlock(&kctx->mmu_lock);
 767         return 0;
 768
 769 fail_unlock:
 770         mutex_unlock(&kctx->mmu_lock);
 771         return err;
 772 }
 773
 774 /*
 775  * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn'
 776  */
 777 int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
 778                                   phys_addr_t *phys, size_t nr,
 779                                   unsigned long flags)
 780 {
 781         int err;
 782
 783         err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags);
 784         kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
 785         return err;
 786 }
 787
 788 KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
 789
 790 /**
 791  * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches
 792  * without retaining the kbase context.
 793  * @kctx: The KBase context.
 794  * @vpfn: The virtual page frame number to start the flush on.
 795  * @nr: The number of pages to flush.
 796  * @sync: Set if the operation should be synchronous or not.
 797  *
 798  * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any
 799  * other locking.
 800  */
 801 static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx,
 802                 u64 vpfn, size_t nr, bool sync)
 803 {
 804         struct kbase_device *kbdev = kctx->kbdev;
 805         int err;
 806         u32 op;
 807
 808         /* Early out if there is nothing to do */
 809         if (nr == 0)
 810                 return;
 811
 812         if (sync)
 813                 op = AS_COMMAND_FLUSH_MEM;
 814         else
 815                 op = AS_COMMAND_FLUSH_PT;
 816
 817         err = kbase_mmu_hw_do_operation(kbdev,
 818                                 &kbdev->as[kctx->as_nr],
 819                                 kctx, vpfn, nr, op, 0);
 820 #if KBASE_GPU_RESET_EN
 821         if (err) {
 822                 /* Flush failed to complete, assume the
 823                  * GPU has hung and perform a reset to
 824                  * recover */
 825                 dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n");
 826
 827                 if (kbase_prepare_to_reset_gpu_locked(kbdev))
 828                         kbase_reset_gpu_locked(kbdev);
 829         }
 830 #endif /* KBASE_GPU_RESET_EN */
 831
 832 #ifndef CONFIG_MALI_NO_MALI
 833         /*
 834          * As this function could be called in interrupt context the sync
 835          * request can't block. Instead log the request and the next flush
 836          * request will pick it up.
 837          */
 838         if ((!err) && sync &&
 839                         kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367))
 840                 atomic_set(&kctx->drain_pending, 1);
 841 #endif /* !CONFIG_MALI_NO_MALI */
 842 }
 843
 844 static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
 845                 u64 vpfn, size_t nr, bool sync)
 846 {
 847         struct kbase_device *kbdev;
 848         bool ctx_is_in_runpool;
 849 #ifndef CONFIG_MALI_NO_MALI
 850         bool drain_pending = false;
 851
 852         if (atomic_xchg(&kctx->drain_pending, 0))
 853                 drain_pending = true;
 854 #endif /* !CONFIG_MALI_NO_MALI */
 855
 856         /* Early out if there is nothing to do */
 857         if (nr == 0)
 858                 return;
 859
 860         kbdev = kctx->kbdev;
 861         ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx);
 862
 863         if (ctx_is_in_runpool) {
 864                 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
 865
 866                 if (!kbase_pm_context_active_handle_suspend(kbdev,
 867                         KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
 868                         int err;
 869                         u32 op;
 870
 871                         /* AS transaction begin */
 872                         mutex_lock(&kbdev->as[
 873                                         kctx->as_nr].transaction_mutex);
 874
 875                         if (sync)
 876                                 op = AS_COMMAND_FLUSH_MEM;
 877                         else
 878                                 op = AS_COMMAND_FLUSH_PT;
 879
 880                         err = kbase_mmu_hw_do_operation(kbdev,
 881                                                 &kbdev->as[kctx->as_nr],
 882                                                 kctx, vpfn, nr, op, 0);
 883
 884 #if KBASE_GPU_RESET_EN
 885                         if (err) {
 886                                 /* Flush failed to complete, assume the
 887                                  * GPU has hung and perform a reset to
 888                                  * recover */
 889                                 dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
 890
 891                                 if (kbase_prepare_to_reset_gpu(kbdev))
 892                                         kbase_reset_gpu(kbdev);
 893                         }
 894 #endif /* KBASE_GPU_RESET_EN */
 895
 896                         mutex_unlock(&kbdev->as[
 897                                         kctx->as_nr].transaction_mutex);
 898                         /* AS transaction end */
 899
 900 #ifndef CONFIG_MALI_NO_MALI
 901                         /*
 902                          * The transaction lock must be dropped before here
 903                          * as kbase_wait_write_flush could take it if
 904                          * the GPU was powered down (static analysis doesn't
 905                          * know this can't happen).
 906                          */
 907                         drain_pending |= (!err) && sync &&
 908                                         kbase_hw_has_issue(kctx->kbdev,
 909                                                         BASE_HW_ISSUE_6367);
 910                         if (drain_pending) {
 911                                 /* Wait for GPU to flush write buffer */
 912                                 kbase_wait_write_flush(kctx);
 913                         }
 914 #endif /* !CONFIG_MALI_NO_MALI */
 915
 916                         kbase_pm_context_idle(kbdev);
 917                 }
 918                 kbasep_js_runpool_release_ctx(kbdev, kctx);
 919         }
 920 }
 921
 922 void kbase_mmu_update(struct kbase_context *kctx)
 923 {
 924         lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
 925         /* ASSERT that the context has a valid as_nr, which is only the case
 926          * when it's scheduled in.
 927          *
 928          * as_nr won't change because the caller has the runpool_irq lock */
 929         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
 930         lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex);
 931
 932         kctx->kbdev->mmu_mode->update(kctx);
 933 }
 934 KBASE_EXPORT_TEST_API(kbase_mmu_update);
 935
 936 void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr)
 937 {
 938         lockdep_assert_held(&kbdev->as[as_nr].transaction_mutex);
 939         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 940
 941         kbdev->mmu_mode->disable_as(kbdev, as_nr);
 942 }
 943
 944 void kbase_mmu_disable(struct kbase_context *kctx)
 945 {
 946         /* ASSERT that the context has a valid as_nr, which is only the case
 947          * when it's scheduled in.
 948          *
 949          * as_nr won't change because the caller has the runpool_irq lock */
 950         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
 951
 952         lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex);
 953         lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
 954
 955         /*
 956          * The address space is being disabled, drain all knowledge of it out
 957          * from the caches as pages and page tables might be freed after this.
 958          *
 959          * The job scheduler code will already be holding the locks and context
 960          * so just do the flush.
 961          */
 962         kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true);
 963
 964         kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr);
 965 }
 966 KBASE_EXPORT_TEST_API(kbase_mmu_disable);
 967
 968 /*
 969  * We actually only discard the ATE, and not the page table
 970  * pages. There is a potential DoS here, as we'll leak memory by
 971  * having PTEs that are potentially unused.  Will require physical
 972  * page accounting, so MMU pages are part of the process allocation.
 973  *
 974  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 975  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 976  * These locks must be taken in the correct order with respect to others
 977  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 978  * information.
 979  */
 980 int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
 981 {
 982         phys_addr_t pgd;
 983         u64 *pgd_page;
 984         struct kbase_device *kbdev;
 985         size_t requested_nr = nr;
 986         struct kbase_mmu_mode const *mmu_mode;
 987         int err;
 988
 989         KBASE_DEBUG_ASSERT(NULL != kctx);
 990         beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr);
 991
 992         if (0 == nr) {
 993                 /* early out if nothing to do */
 994                 return 0;
 995         }
 996
 997         mutex_lock(&kctx->mmu_lock);
 998
 999         kbdev = kctx->kbdev;
1000         mmu_mode = kbdev->mmu_mode;
1001
1002         while (nr) {
1003                 unsigned int i;
1004                 unsigned int index = vpfn & 0x1FF;
1005                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
1006                 struct page *p;
1007
1008                 if (count > nr)
1009                         count = nr;
1010
1011                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
1012                 if (!pgd) {
1013                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n");
1014                         err = -EINVAL;
1015                         goto fail_unlock;
1016                 }
1017
1018                 p = pfn_to_page(PFN_DOWN(pgd));
1019                 pgd_page = kmap(p);
1020                 if (!pgd_page) {
1021                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n");
1022                         err = -ENOMEM;
1023                         goto fail_unlock;
1024                 }
1025
1026                 for (i = 0; i < count; i++)
1027                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
1028
1029                 vpfn += count;
1030                 nr -= count;
1031
1032                 kbase_mmu_sync_pgd(kctx->kbdev,
1033                                 kbase_dma_addr(p) + (index * sizeof(u64)),
1034                                 count * sizeof(u64));
1035
1036                 kunmap(p);
1037         }
1038
1039         mutex_unlock(&kctx->mmu_lock);
1040         kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1041         return 0;
1042
1043 fail_unlock:
1044         mutex_unlock(&kctx->mmu_lock);
1045         kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1046         return err;
1047 }
1048
1049 KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
1050
1051 /**
1052  * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'.
1053  * This call is being triggered as a response to the changes of the mem attributes
1054  *
1055  * @pre : The caller is responsible for validating the memory attributes
1056  *
1057  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
1058  * currently scheduled into the runpool, and so potentially uses a lot of locks.
1059  * These locks must be taken in the correct order with respect to others
1060  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
1061  * information.
1062  */
1063 int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags)
1064 {
1065         phys_addr_t pgd;
1066         u64 *pgd_page;
1067         size_t requested_nr = nr;
1068         struct kbase_mmu_mode const *mmu_mode;
1069         int err;
1070
1071         KBASE_DEBUG_ASSERT(NULL != kctx);
1072         KBASE_DEBUG_ASSERT(0 != vpfn);
1073         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
1074
1075         /* Early out if there is nothing to do */
1076         if (nr == 0)
1077                 return 0;
1078
1079         mutex_lock(&kctx->mmu_lock);
1080
1081         mmu_mode = kctx->kbdev->mmu_mode;
1082
1083         dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages",
1084                         vpfn, phys, nr);
1085
1086         while (nr) {
1087                 unsigned int i;
1088                 unsigned int index = vpfn & 0x1FF;
1089                 size_t count = KBASE_MMU_PAGE_ENTRIES - index;
1090                 struct page *p;
1091
1092                 if (count > nr)
1093                         count = nr;
1094
1095                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
1096                 if (!pgd) {
1097                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n");
1098                         err = -EINVAL;
1099                         goto fail_unlock;
1100                 }
1101
1102                 p = pfn_to_page(PFN_DOWN(pgd));
1103                 pgd_page = kmap(p);
1104                 if (!pgd_page) {
1105                         dev_warn(kctx->kbdev->dev, "kmap failure\n");
1106                         err = -ENOMEM;
1107                         goto fail_unlock;
1108                 }
1109
1110                 for (i = 0; i < count; i++)
1111                         mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i],
1112                                         flags);
1113
1114                 phys += count;
1115                 vpfn += count;
1116                 nr -= count;
1117
1118                 kbase_mmu_sync_pgd(kctx->kbdev,
1119                                 kbase_dma_addr(p) + (index * sizeof(u64)),
1120                                 count * sizeof(u64));
1121
1122                 kunmap(pfn_to_page(PFN_DOWN(pgd)));
1123         }
1124
1125         mutex_unlock(&kctx->mmu_lock);
1126         kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1127         return 0;
1128
1129 fail_unlock:
1130         mutex_unlock(&kctx->mmu_lock);
1131         kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1132         return err;
1133 }
1134
1135 /* This is a debug feature only */
1136 static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd)
1137 {
1138         u64 *page;
1139         int i;
1140
1141         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
1142         /* kmap_atomic should NEVER fail. */
1143         KBASE_DEBUG_ASSERT(NULL != page);
1144
1145         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1146                 if (kctx->kbdev->mmu_mode->ate_is_valid(page[i]))
1147                         beenthere(kctx, "live pte %016lx", (unsigned long)page[i]);
1148         }
1149         kunmap_atomic(page);
1150 }
1151
1152 static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer)
1153 {
1154         phys_addr_t target_pgd;
1155         u64 *pgd_page;
1156         int i;
1157         struct kbase_mmu_mode const *mmu_mode;
1158
1159         KBASE_DEBUG_ASSERT(NULL != kctx);
1160         lockdep_assert_held(&kctx->mmu_lock);
1161
1162         pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
1163         /* kmap_atomic should NEVER fail. */
1164         KBASE_DEBUG_ASSERT(NULL != pgd_page);
1165         /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */
1166         memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE);
1167         kunmap_atomic(pgd_page);
1168         pgd_page = pgd_page_buffer;
1169
1170         mmu_mode = kctx->kbdev->mmu_mode;
1171
1172         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1173                 target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
1174
1175                 if (target_pgd) {
1176                         if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) {
1177                                 mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64)));
1178                         } else {
1179                                 /*
1180                                  * So target_pte is a level-3 page.
1181                                  * As a leaf, it is safe to free it.
1182                                  * Unless we have live pages attached to it!
1183                                  */
1184                                 mmu_check_unused(kctx, target_pgd);
1185                         }
1186
1187                         beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1);
1188                         if (zap) {
1189                                 struct page *p = phys_to_page(target_pgd);
1190
1191                                 kbase_mem_pool_free(&kctx->mem_pool, p, true);
1192                                 kbase_process_page_usage_dec(kctx, 1);
1193                                 kbase_atomic_sub_pages(1, &kctx->used_pages);
1194                                 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1195                         }
1196                 }
1197         }
1198 }
1199
1200 int kbase_mmu_init(struct kbase_context *kctx)
1201 {
1202         KBASE_DEBUG_ASSERT(NULL != kctx);
1203         KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages);
1204
1205         mutex_init(&kctx->mmu_lock);
1206
1207         /* Preallocate MMU depth of four pages for mmu_teardown_level to use */
1208         kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
1209
1210         if (NULL == kctx->mmu_teardown_pages)
1211                 return -ENOMEM;
1212
1213         return 0;
1214 }
1215
1216 void kbase_mmu_term(struct kbase_context *kctx)
1217 {
1218         KBASE_DEBUG_ASSERT(NULL != kctx);
1219         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1220
1221         kfree(kctx->mmu_teardown_pages);
1222         kctx->mmu_teardown_pages = NULL;
1223 }
1224
1225 void kbase_mmu_free_pgd(struct kbase_context *kctx)
1226 {
1227         int new_page_count __maybe_unused;
1228
1229         KBASE_DEBUG_ASSERT(NULL != kctx);
1230         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1231
1232         mutex_lock(&kctx->mmu_lock);
1233         mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages);
1234         mutex_unlock(&kctx->mmu_lock);
1235
1236         beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd);
1237         kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true);
1238         kbase_process_page_usage_dec(kctx, 1);
1239         new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages);
1240         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1241
1242         kbase_tlstream_aux_pagesalloc(
1243                         (u32)kctx->id,
1244                         (u64)new_page_count);
1245 }
1246
1247 KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd);
1248
1249 static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left)
1250 {
1251         phys_addr_t target_pgd;
1252         u64 *pgd_page;
1253         int i;
1254         size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64);
1255         size_t dump_size;
1256         struct kbase_mmu_mode const *mmu_mode;
1257
1258         KBASE_DEBUG_ASSERT(NULL != kctx);
1259         lockdep_assert_held(&kctx->mmu_lock);
1260
1261         mmu_mode = kctx->kbdev->mmu_mode;
1262
1263         pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd)));
1264         if (!pgd_page) {
1265                 dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n");
1266                 return 0;
1267         }
1268
1269         if (*size_left >= size) {
1270                 /* A modified physical address that contains the page table level */
1271                 u64 m_pgd = pgd | level;
1272
1273                 /* Put the modified physical address in the output buffer */
1274                 memcpy(*buffer, &m_pgd, sizeof(m_pgd));
1275                 *buffer += sizeof(m_pgd);
1276
1277                 /* Followed by the page table itself */
1278                 memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES);
1279                 *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES;
1280
1281                 *size_left -= size;
1282         }
1283
1284         if (level < MIDGARD_MMU_BOTTOMLEVEL) {
1285                 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1286                         if (mmu_mode->pte_is_valid(pgd_page[i])) {
1287                                 target_pgd = mmu_mode->pte_to_phy_addr(
1288                                                 pgd_page[i]);
1289
1290                                 dump_size = kbasep_mmu_dump_level(kctx,
1291                                                 target_pgd, level + 1,
1292                                                 buffer, size_left);
1293                                 if (!dump_size) {
1294                                         kunmap(pfn_to_page(PFN_DOWN(pgd)));
1295                                         return 0;
1296                                 }
1297                                 size += dump_size;
1298                         }
1299                 }
1300         }
1301
1302         kunmap(pfn_to_page(PFN_DOWN(pgd)));
1303
1304         return size;
1305 }
1306
1307 void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages)
1308 {
1309         void *kaddr;
1310         size_t size_left;
1311
1312         KBASE_DEBUG_ASSERT(kctx);
1313
1314         if (0 == nr_pages) {
1315                 /* can't dump in a 0 sized buffer, early out */
1316                 return NULL;
1317         }
1318
1319         mutex_lock(&kctx->mmu_lock);
1320
1321         size_left = nr_pages * PAGE_SIZE;
1322
1323         KBASE_DEBUG_ASSERT(0 != size_left);
1324         kaddr = vmalloc_user(size_left);
1325
1326         if (kaddr) {
1327                 u64 end_marker = 0xFFULL;
1328                 char *buffer;
1329                 char *mmu_dump_buffer;
1330                 u64 config[3];
1331                 size_t size;
1332
1333                 buffer = (char *)kaddr;
1334                 mmu_dump_buffer = buffer;
1335
1336                 if (kctx->api_version >= KBASE_API_VERSION(8, 4)) {
1337                         struct kbase_mmu_setup as_setup;
1338
1339                         kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup);
1340                         config[0] = as_setup.transtab;
1341                         config[1] = as_setup.memattr;
1342                         config[2] = as_setup.transcfg;
1343                         memcpy(buffer, &config, sizeof(config));
1344                         mmu_dump_buffer += sizeof(config);
1345                         size_left -= sizeof(config);
1346                 }
1347
1348
1349
1350                 size = kbasep_mmu_dump_level(kctx,
1351                                 kctx->pgd,
1352                                 MIDGARD_MMU_TOPLEVEL,
1353                                 &mmu_dump_buffer,
1354                                 &size_left);
1355
1356                 if (!size)
1357                         goto fail_free;
1358
1359                 /* Add on the size for the end marker */
1360                 size += sizeof(u64);
1361                 /* Add on the size for the config */
1362                 if (kctx->api_version >= KBASE_API_VERSION(8, 4))
1363                         size += sizeof(config);
1364
1365
1366                 if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) {
1367                         /* The buffer isn't big enough - free the memory and return failure */
1368                         goto fail_free;
1369                 }
1370
1371                 /* Add the end marker */
1372                 memcpy(mmu_dump_buffer, &end_marker, sizeof(u64));
1373         }
1374
1375         mutex_unlock(&kctx->mmu_lock);
1376         return kaddr;
1377
1378 fail_free:
1379         vfree(kaddr);
1380         mutex_unlock(&kctx->mmu_lock);
1381         return NULL;
1382 }
1383 KBASE_EXPORT_TEST_API(kbase_mmu_dump);
1384
1385 void bus_fault_worker(struct work_struct *data)
1386 {
1387         struct kbase_as *faulting_as;
1388         int as_no;
1389         struct kbase_context *kctx;
1390         struct kbase_device *kbdev;
1391 #if KBASE_GPU_RESET_EN
1392         bool reset_status = false;
1393 #endif /* KBASE_GPU_RESET_EN */
1394
1395         faulting_as = container_of(data, struct kbase_as, work_busfault);
1396
1397         as_no = faulting_as->number;
1398
1399         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
1400
1401         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
1402          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
1403          */
1404         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
1405         if (WARN_ON(!kctx)) {
1406                 atomic_dec(&kbdev->faults_pending);
1407                 return;
1408         }
1409
1410 #if KBASE_GPU_RESET_EN
1411         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1412                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1413                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1414                  * are evicted from the GPU before the switch.
1415                  */
1416                 dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1417                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1418         }
1419 #endif /* KBASE_GPU_RESET_EN */
1420         /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */
1421         if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1422                 unsigned long flags;
1423
1424                 /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1425                 /* AS transaction begin */
1426                 mutex_lock(&kbdev->as[as_no].transaction_mutex);
1427
1428                 /* Set the MMU into unmapped mode */
1429                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1430                 kbase_mmu_disable(kctx);
1431                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock,
1432                                 flags);
1433
1434                 mutex_unlock(&kbdev->as[as_no].transaction_mutex);
1435                 /* AS transaction end */
1436
1437                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
1438                                          KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1439                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
1440                                          KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1441
1442                 kbase_pm_context_idle(kbdev);
1443         }
1444
1445 #if KBASE_GPU_RESET_EN
1446         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1447                 kbase_reset_gpu(kbdev);
1448 #endif /* KBASE_GPU_RESET_EN */
1449
1450         kbasep_js_runpool_release_ctx(kbdev, kctx);
1451
1452         atomic_dec(&kbdev->faults_pending);
1453 }
1454
1455 const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
1456 {
1457         const char *e;
1458
1459         switch (exception_code) {
1460                 /* Non-Fault Status code */
1461         case 0x00:
1462                 e = "NOT_STARTED/IDLE/OK";
1463                 break;
1464         case 0x01:
1465                 e = "DONE";
1466                 break;
1467         case 0x02:
1468                 e = "INTERRUPTED";
1469                 break;
1470         case 0x03:
1471                 e = "STOPPED";
1472                 break;
1473         case 0x04:
1474                 e = "TERMINATED";
1475                 break;
1476         case 0x08:
1477                 e = "ACTIVE";
1478                 break;
1479                 /* Job exceptions */
1480         case 0x40:
1481                 e = "JOB_CONFIG_FAULT";
1482                 break;
1483         case 0x41:
1484                 e = "JOB_POWER_FAULT";
1485                 break;
1486         case 0x42:
1487                 e = "JOB_READ_FAULT";
1488                 break;
1489         case 0x43:
1490                 e = "JOB_WRITE_FAULT";
1491                 break;
1492         case 0x44:
1493                 e = "JOB_AFFINITY_FAULT";
1494                 break;
1495         case 0x48:
1496                 e = "JOB_BUS_FAULT";
1497                 break;
1498         case 0x50:
1499                 e = "INSTR_INVALID_PC";
1500                 break;
1501         case 0x51:
1502                 e = "INSTR_INVALID_ENC";
1503                 break;
1504         case 0x52:
1505                 e = "INSTR_TYPE_MISMATCH";
1506                 break;
1507         case 0x53:
1508                 e = "INSTR_OPERAND_FAULT";
1509                 break;
1510         case 0x54:
1511                 e = "INSTR_TLS_FAULT";
1512                 break;
1513         case 0x55:
1514                 e = "INSTR_BARRIER_FAULT";
1515                 break;
1516         case 0x56:
1517                 e = "INSTR_ALIGN_FAULT";
1518                 break;
1519         case 0x58:
1520                 e = "DATA_INVALID_FAULT";
1521                 break;
1522         case 0x59:
1523                 e = "TILE_RANGE_FAULT";
1524                 break;
1525         case 0x5A:
1526                 e = "ADDR_RANGE_FAULT";
1527                 break;
1528         case 0x60:
1529                 e = "OUT_OF_MEMORY";
1530                 break;
1531                 /* GPU exceptions */
1532         case 0x80:
1533                 e = "DELAYED_BUS_FAULT";
1534                 break;
1535         case 0x88:
1536                 e = "SHAREABILITY_FAULT";
1537                 break;
1538                 /* MMU exceptions */
1539         case 0xC0:
1540         case 0xC1:
1541         case 0xC2:
1542         case 0xC3:
1543         case 0xC4:
1544         case 0xC5:
1545         case 0xC6:
1546         case 0xC7:
1547                 e = "TRANSLATION_FAULT";
1548                 break;
1549         case 0xC8:
1550 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
1551         case 0xC9:
1552         case 0xCA:
1553         case 0xCB:
1554         case 0xCC:
1555         case 0xCD:
1556         case 0xCE:
1557         case 0xCF:
1558 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
1559                 e = "PERMISSION_FAULT";
1560                 break;
1561         case 0xD0:
1562         case 0xD1:
1563         case 0xD2:
1564         case 0xD3:
1565         case 0xD4:
1566         case 0xD5:
1567         case 0xD6:
1568         case 0xD7:
1569                 e = "TRANSTAB_BUS_FAULT";
1570                 break;
1571         case 0xD8:
1572 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
1573         case 0xD9:
1574         case 0xDA:
1575         case 0xDB:
1576         case 0xDC:
1577         case 0xDD:
1578         case 0xDE:
1579         case 0xDF:
1580 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
1581                 e = "ACCESS_FLAG";
1582                 break;
1583 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
1584         case 0xE0:
1585         case 0xE1:
1586         case 0xE2:
1587         case 0xE3:
1588         case 0xE4:
1589         case 0xE5:
1590         case 0xE6:
1591         case 0xE7:
1592                 e = "ADDRESS_SIZE_FAULT";
1593                 break;
1594         case 0xE8:
1595         case 0xE9:
1596         case 0xEA:
1597         case 0xEB:
1598         case 0xEC:
1599         case 0xED:
1600         case 0xEE:
1601         case 0xEF:
1602                 e = "MEMORY_ATTRIBUTES_FAULT";
1603 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
1604                 break;
1605         default:
1606                 e = "UNKNOWN";
1607                 break;
1608         };
1609
1610         return e;
1611 }
1612
1613 static const char *access_type_name(struct kbase_device *kbdev,
1614                 u32 fault_status)
1615 {
1616         switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1617         case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
1618 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
1619                 return "ATOMIC";
1620 #else
1621                 return "UNKNOWN";
1622 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
1623         case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1624                 return "READ";
1625         case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1626                 return "WRITE";
1627         case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1628                 return "EXECUTE";
1629         default:
1630                 WARN_ON(1);
1631                 return NULL;
1632         }
1633 }
1634
1635 /**
1636  * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on.
1637  */
1638 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
1639                 struct kbase_as *as, const char *reason_str)
1640 {
1641         unsigned long flags;
1642         int exception_type;
1643         int access_type;
1644         int source_id;
1645         int as_no;
1646         struct kbase_device *kbdev;
1647         struct kbasep_js_device_data *js_devdata;
1648
1649 #if KBASE_GPU_RESET_EN
1650         bool reset_status = false;
1651 #endif
1652
1653         as_no = as->number;
1654         kbdev = kctx->kbdev;
1655         js_devdata = &kbdev->js_data;
1656
1657         /* ASSERT that the context won't leave the runpool */
1658         KBASE_DEBUG_ASSERT(kbasep_js_debug_check_ctx_refcount(kbdev, kctx) > 0);
1659
1660         /* decode the fault status */
1661         exception_type = as->fault_status & 0xFF;
1662         access_type = (as->fault_status >> 8) & 0x3;
1663         source_id = (as->fault_status >> 16);
1664
1665         /* terminal fault, print info about the fault */
1666         dev_err(kbdev->dev,
1667                 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
1668                 "Reason: %s\n"
1669                 "raw fault status 0x%X\n"
1670                 "decoded fault status: %s\n"
1671                 "exception type 0x%X: %s\n"
1672                 "access type 0x%X: %s\n"
1673                 "source id 0x%X\n"
1674                 "pid: %d\n",
1675                 as_no, as->fault_addr,
1676                 reason_str,
1677                 as->fault_status,
1678                 (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1679                 exception_type, kbase_exception_name(kbdev, exception_type),
1680                 access_type, access_type_name(kbdev, as->fault_status),
1681                 source_id,
1682                 kctx->pid);
1683
1684         /* hardware counters dump fault handling */
1685         if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) &&
1686                         (kbdev->hwcnt.backend.state ==
1687                                                 KBASE_INSTR_STATE_DUMPING)) {
1688                 unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
1689
1690                 if ((as->fault_addr >= kbdev->hwcnt.addr) &&
1691                                 (as->fault_addr < (kbdev->hwcnt.addr +
1692                                                 (num_core_groups * 2048))))
1693                         kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT;
1694         }
1695
1696         /* Stop the kctx from submitting more jobs and cause it to be scheduled
1697          * out/rescheduled - this will occur on releasing the context's refcount */
1698         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1699         kbasep_js_clear_submit_allowed(js_devdata, kctx);
1700         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1701
1702         /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this
1703          * context can appear in the job slots from this point on */
1704         kbase_backend_jm_kill_jobs_from_kctx(kctx);
1705         /* AS transaction begin */
1706         mutex_lock(&as->transaction_mutex);
1707 #if KBASE_GPU_RESET_EN
1708         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1709                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1710                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1711                  * are evicted from the GPU before the switch.
1712                  */
1713                 dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery.");
1714                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1715         }
1716 #endif /* KBASE_GPU_RESET_EN */
1717         /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1718         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1719         kbase_mmu_disable(kctx);
1720         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1721
1722         mutex_unlock(&as->transaction_mutex);
1723         /* AS transaction end */
1724         /* Clear down the fault */
1725         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1726                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1727         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1728                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1729
1730 #if KBASE_GPU_RESET_EN
1731         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1732                 kbase_reset_gpu(kbdev);
1733 #endif /* KBASE_GPU_RESET_EN */
1734 }
1735
1736 void kbasep_as_do_poke(struct work_struct *work)
1737 {
1738         struct kbase_as *as;
1739         struct kbase_device *kbdev;
1740         struct kbase_context *kctx;
1741         unsigned long flags;
1742
1743         KBASE_DEBUG_ASSERT(work);
1744         as = container_of(work, struct kbase_as, poke_work);
1745         kbdev = container_of(as, struct kbase_device, as[as->number]);
1746         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1747
1748         /* GPU power will already be active by virtue of the caller holding a JS
1749          * reference on the address space, and will not release it until this worker
1750          * has finished */
1751
1752         /* Further to the comment above, we know that while this function is running
1753          * the AS will not be released as before the atom is released this workqueue
1754          * is flushed (in kbase_as_poking_timer_release_atom)
1755          */
1756         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number);
1757
1758         /* AS transaction begin */
1759         mutex_lock(&as->transaction_mutex);
1760         /* Force a uTLB invalidate */
1761         kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0,
1762                                   AS_COMMAND_UNLOCK, 0);
1763         mutex_unlock(&as->transaction_mutex);
1764         /* AS transaction end */
1765
1766         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1767         if (as->poke_refcount &&
1768                 !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) {
1769                 /* Only queue up the timer if we need it, and we're not trying to kill it */
1770                 hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL);
1771         }
1772         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1773 }
1774
1775 enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer)
1776 {
1777         struct kbase_as *as;
1778         int queue_work_ret;
1779
1780         KBASE_DEBUG_ASSERT(NULL != timer);
1781         as = container_of(timer, struct kbase_as, poke_timer);
1782         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1783
1784         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1785         KBASE_DEBUG_ASSERT(queue_work_ret);
1786         return HRTIMER_NORESTART;
1787 }
1788
1789 /**
1790  * Retain the poking timer on an atom's context (if the atom hasn't already
1791  * done so), and start the timer (if it's not already started).
1792  *
1793  * This must only be called on a context that's scheduled in, and an atom
1794  * that's running on the GPU.
1795  *
1796  * The caller must hold kbasep_js_device_data::runpool_irq::lock
1797  *
1798  * This can be called safely from atomic context
1799  */
1800 void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1801 {
1802         struct kbase_as *as;
1803
1804         KBASE_DEBUG_ASSERT(kbdev);
1805         KBASE_DEBUG_ASSERT(kctx);
1806         KBASE_DEBUG_ASSERT(katom);
1807         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1808         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1809
1810         if (katom->poking)
1811                 return;
1812
1813         katom->poking = 1;
1814
1815         /* It's safe to work on the as/as_nr without an explicit reference,
1816          * because the caller holds the runpool_irq lock, and the atom itself
1817          * was also running and had already taken a reference  */
1818         as = &kbdev->as[kctx->as_nr];
1819
1820         if (++(as->poke_refcount) == 1) {
1821                 /* First refcount for poke needed: check if not already in flight */
1822                 if (!as->poke_state) {
1823                         /* need to start poking */
1824                         as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT;
1825                         queue_work(as->poke_wq, &as->poke_work);
1826                 }
1827         }
1828 }
1829
1830 /**
1831  * If an atom holds a poking timer, release it and wait for it to finish
1832  *
1833  * This must only be called on a context that's scheduled in, and an atom
1834  * that still has a JS reference on the context
1835  *
1836  * This must \b not be called from atomic context, since it can sleep.
1837  */
1838 void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1839 {
1840         struct kbase_as *as;
1841         unsigned long flags;
1842
1843         KBASE_DEBUG_ASSERT(kbdev);
1844         KBASE_DEBUG_ASSERT(kctx);
1845         KBASE_DEBUG_ASSERT(katom);
1846         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1847
1848         if (!katom->poking)
1849                 return;
1850
1851         as = &kbdev->as[kctx->as_nr];
1852
1853         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1854         KBASE_DEBUG_ASSERT(as->poke_refcount > 0);
1855         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1856
1857         if (--(as->poke_refcount) == 0) {
1858                 as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE;
1859                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1860
1861                 hrtimer_cancel(&as->poke_timer);
1862                 flush_workqueue(as->poke_wq);
1863
1864                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1865
1866                 /* Re-check whether it's still needed */
1867                 if (as->poke_refcount) {
1868                         int queue_work_ret;
1869                         /* Poking still needed:
1870                          * - Another retain will not be starting the timer or queueing work,
1871                          * because it's still marked as in-flight
1872                          * - The hrtimer has finished, and has not started a new timer or
1873                          * queued work because it's been marked as killing
1874                          *
1875                          * So whatever happens now, just queue the work again */
1876                         as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE);
1877                         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1878                         KBASE_DEBUG_ASSERT(queue_work_ret);
1879                 } else {
1880                         /* It isn't - so mark it as not in flight, and not killing */
1881                         as->poke_state = 0u;
1882
1883                         /* The poke associated with the atom has now finished. If this is
1884                          * also the last atom on the context, then we can guarentee no more
1885                          * pokes (and thus no more poking register accesses) will occur on
1886                          * the context until new atoms are run */
1887                 }
1888         }
1889         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1890
1891         katom->poking = 0;
1892 }
1893
1894 void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as)
1895 {
1896         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1897
1898         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1899
1900         if (!kctx) {
1901                 dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n",
1902                                  kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault",
1903                                  as->number, as->fault_addr);
1904
1905                 /* Since no ctx was found, the MMU must be disabled. */
1906                 WARN_ON(as->current_setup.transtab);
1907
1908                 if (kbase_as_has_bus_fault(as)) {
1909                         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1910                                         KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1911                         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1912                                         KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1913                 } else if (kbase_as_has_page_fault(as)) {
1914                         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1915                                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1916                         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1917                                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1918                 }
1919
1920 #if KBASE_GPU_RESET_EN
1921                 if (kbase_as_has_bus_fault(as) &&
1922                                 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1923                         bool reset_status;
1924                         /*
1925                          * Reset the GPU, like in bus_fault_worker, in case an
1926                          * earlier error hasn't been properly cleared by this
1927                          * point.
1928                          */
1929                         dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1930                         reset_status = kbase_prepare_to_reset_gpu_locked(kbdev);
1931                         if (reset_status)
1932                                 kbase_reset_gpu_locked(kbdev);
1933                 }
1934 #endif /* KBASE_GPU_RESET_EN */
1935
1936                 return;
1937         }
1938
1939         if (kbase_as_has_bus_fault(as)) {
1940                 /*
1941                  * hw counters dumping in progress, signal the
1942                  * other thread that it failed
1943                  */
1944                 if ((kbdev->hwcnt.kctx == kctx) &&
1945                     (kbdev->hwcnt.backend.state ==
1946                                         KBASE_INSTR_STATE_DUMPING))
1947                         kbdev->hwcnt.backend.state =
1948                                                 KBASE_INSTR_STATE_FAULT;
1949
1950                 /*
1951                  * Stop the kctx from submitting more jobs and cause it
1952                  * to be scheduled out/rescheduled when all references
1953                  * to it are released
1954                  */
1955                 kbasep_js_clear_submit_allowed(js_devdata, kctx);
1956
1957 #ifdef CONFIG_MALI_GPU_MMU_AARCH64
1958                 dev_warn(kbdev->dev,
1959                                 "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n",
1960                                 as->number, as->fault_addr,
1961                                 as->fault_extra_addr);
1962 #else
1963                 dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
1964                                 as->number, as->fault_addr);
1965 #endif /* CONFIG_MALI_GPU_MMU_AARCH64 */
1966
1967                 /*
1968                  * We need to switch to UNMAPPED mode - but we do this in a
1969                  * worker so that we can sleep
1970                  */
1971                 kbdev->kbase_group_error++;
1972                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault));
1973                 WARN_ON(work_pending(&as->work_busfault));
1974                 queue_work(as->pf_wq, &as->work_busfault);
1975                 atomic_inc(&kbdev->faults_pending);
1976         } else {
1977                 kbdev->kbase_group_error++;
1978                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault));
1979                 WARN_ON(work_pending(&as->work_pagefault));
1980                 queue_work(as->pf_wq, &as->work_pagefault);
1981                 atomic_inc(&kbdev->faults_pending);
1982         }
1983 }
1984
1985 void kbase_flush_mmu_wqs(struct kbase_device *kbdev)
1986 {
1987         int i;
1988
1989         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
1990                 struct kbase_as *as = &kbdev->as[i];
1991
1992                 flush_workqueue(as->pf_wq);
1993         }
1994 }