drivers/gpu/arm/midgard_for_linux/mali_kbase_mmu.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_mmu.c
  22  * Base kernel MMU management.
  23  */
  24
  25 /* #define DEBUG    1 */
  26 #include <linux/kernel.h>
  27 #include <linux/dma-mapping.h>
  28 #include <mali_kbase.h>
  29 #include <mali_midg_regmap.h>
  30 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  31 #include <mali_kbase_gator.h>
  32 #endif
  33 #if defined(CONFIG_MALI_MIPE_ENABLED)
  34 #include <mali_kbase_tlstream.h>
  35 #endif
  36 #include <mali_kbase_debug.h>
  37
  38 #define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  39
  40 #include <mali_kbase_defs.h>
  41 #include <mali_kbase_hw.h>
  42 #include <mali_kbase_mmu_hw.h>
  43 #include <mali_kbase_hwaccess_jm.h>
  44
  45 #define KBASE_MMU_PAGE_ENTRIES 512
  46
  47 /**
  48  * kbase_mmu_sync_pgd - sync page directory to memory
  49  * @kbdev:      Device pointer.
  50  * @handle:     Address of DMA region.
  51  * @size:       Size of the region to sync.
  52  *
  53  * This should be called after each page directory update.
  54  */
  55
  56 static void kbase_mmu_sync_pgd(struct kbase_device *kbdev,
  57                 dma_addr_t handle, size_t size)
  58 {
  59
  60         dma_sync_single_for_device(kbdev->dev, handle, size, DMA_TO_DEVICE);
  61 }
  62
  63 /*
  64  * Definitions:
  65  * - PGD: Page Directory.
  66  * - PTE: Page Table Entry. A 64bit value pointing to the next
  67  *        level of translation
  68  * - ATE: Address Transation Entry. A 64bit value pointing to
  69  *        a 4kB physical page.
  70  */
  71
  72 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
  73                 struct kbase_as *as, const char *reason_str);
  74
  75
  76 static size_t make_multiple(size_t minimum, size_t multiple)
  77 {
  78         size_t remainder = minimum % multiple;
  79
  80         if (remainder == 0)
  81                 return minimum;
  82
  83         return minimum + multiple - remainder;
  84 }
  85
  86 void page_fault_worker(struct work_struct *data)
  87 {
  88         u64 fault_pfn;
  89         u32 fault_status;
  90         size_t new_pages;
  91         size_t fault_rel_pfn;
  92         struct kbase_as *faulting_as;
  93         int as_no;
  94         struct kbase_context *kctx;
  95         struct kbase_device *kbdev;
  96         struct kbase_va_region *region;
  97         int err;
  98         bool grown = false;
  99
 100         faulting_as = container_of(data, struct kbase_as, work_pagefault);
 101         fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
 102         as_no = faulting_as->number;
 103
 104         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
 105
 106         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
 107          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
 108          */
 109         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
 110         if (WARN_ON(!kctx)) {
 111                 atomic_dec(&kbdev->faults_pending);
 112                 return;
 113         }
 114
 115         KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
 116
 117         fault_status = faulting_as->fault_status;
 118         switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
 119
 120         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
 121                 /* need to check against the region to handle this one */
 122                 break;
 123
 124         case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
 125                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 126                                 "Permission failure");
 127                 goto fault_done;
 128
 129         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
 130                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 131                                 "Tranlation table bus fault");
 132                 goto fault_done;
 133
 134         case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
 135                 /* nothing to do, but we don't expect this fault currently */
 136                 dev_warn(kbdev->dev, "Access flag unexpectedly set");
 137                 goto fault_done;
 138
 139
 140         default:
 141                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 142                                 "Unknown fault code");
 143                 goto fault_done;
 144         }
 145
 146         /* so we have a translation fault, let's see if it is for growable
 147          * memory */
 148         kbase_gpu_vm_lock(kctx);
 149
 150         region = kbase_region_tracker_find_region_enclosing_address(kctx,
 151                         faulting_as->fault_addr);
 152         if (!region || region->flags & KBASE_REG_FREE) {
 153                 kbase_gpu_vm_unlock(kctx);
 154                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 155                                 "Memory is not mapped on the GPU");
 156                 goto fault_done;
 157         }
 158
 159         if ((region->flags & GROWABLE_FLAGS_REQUIRED)
 160                         != GROWABLE_FLAGS_REQUIRED) {
 161                 kbase_gpu_vm_unlock(kctx);
 162                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 163                                 "Memory is not growable");
 164                 goto fault_done;
 165         }
 166
 167         /* find the size we need to grow it by */
 168         /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
 169          * validating the fault_adress to be within a size_t from the start_pfn */
 170         fault_rel_pfn = fault_pfn - region->start_pfn;
 171
 172         if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
 173                 dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
 174                                 faulting_as->fault_addr, region->start_pfn,
 175                                 region->start_pfn +
 176                                 kbase_reg_current_backed_size(region));
 177
 178                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 179                                 KBASE_MMU_FAULT_TYPE_PAGE);
 180                 /* [1] in case another page fault occurred while we were
 181                  * handling the (duplicate) page fault we need to ensure we
 182                  * don't loose the other page fault as result of us clearing
 183                  * the MMU IRQ. Therefore, after we clear the MMU IRQ we send
 184                  * an UNLOCK command that will retry any stalled memory
 185                  * transaction (which should cause the other page fault to be
 186                  * raised again).
 187                  */
 188                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 189                                 AS_COMMAND_UNLOCK, 1);
 190                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 191                                 KBASE_MMU_FAULT_TYPE_PAGE);
 192                 kbase_gpu_vm_unlock(kctx);
 193
 194                 goto fault_done;
 195         }
 196
 197         new_pages = make_multiple(fault_rel_pfn -
 198                         kbase_reg_current_backed_size(region) + 1,
 199                         region->extent);
 200
 201         /* cap to max vsize */
 202         if (new_pages + kbase_reg_current_backed_size(region) >
 203                         region->nr_pages)
 204                 new_pages = region->nr_pages -
 205                                 kbase_reg_current_backed_size(region);
 206
 207         if (0 == new_pages) {
 208                 /* Duplicate of a fault we've already handled, nothing to do */
 209                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 210                                 KBASE_MMU_FAULT_TYPE_PAGE);
 211                 /* See comment [1] about UNLOCK usage */
 212                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 213                                 AS_COMMAND_UNLOCK, 1);
 214                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 215                                 KBASE_MMU_FAULT_TYPE_PAGE);
 216                 kbase_gpu_vm_unlock(kctx);
 217                 goto fault_done;
 218         }
 219
 220         if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
 221                 if (region->gpu_alloc != region->cpu_alloc) {
 222                         if (kbase_alloc_phy_pages_helper(
 223                                         region->cpu_alloc, new_pages) == 0) {
 224                                 grown = true;
 225                         } else {
 226                                 kbase_free_phy_pages_helper(region->gpu_alloc,
 227                                                 new_pages);
 228                         }
 229                 } else {
 230                         grown = true;
 231                 }
 232         }
 233
 234
 235         if (grown) {
 236                 u32 op;
 237
 238                 /* alloc success */
 239                 KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
 240
 241                 /* AS transaction begin */
 242                 mutex_lock(&faulting_as->transaction_mutex);
 243
 244                 /* set up the new pages */
 245                 err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
 246                 if (err) {
 247                         /* failed to insert pages, handle as a normal PF */
 248                         mutex_unlock(&faulting_as->transaction_mutex);
 249                         kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
 250                         if (region->gpu_alloc != region->cpu_alloc)
 251                                 kbase_free_phy_pages_helper(region->cpu_alloc,
 252                                                 new_pages);
 253                         kbase_gpu_vm_unlock(kctx);
 254                         /* The locked VA region will be unlocked and the cache invalidated in here */
 255                         kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 256                                         "Page table update failure");
 257                         goto fault_done;
 258                 }
 259 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 260                 kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
 261 #endif
 262 #if defined(CONFIG_MALI_MIPE_ENABLED)
 263                 kbase_tlstream_aux_pagefault(kctx->id, (u64)new_pages);
 264 #endif
 265
 266                 /* flush L2 and unlock the VA (resumes the MMU) */
 267                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
 268                         op = AS_COMMAND_FLUSH;
 269                 else
 270                         op = AS_COMMAND_FLUSH_PT;
 271
 272                 /* clear MMU interrupt - this needs to be done after updating
 273                  * the page tables but before issuing a FLUSH command. The
 274                  * FLUSH cmd has a side effect that it restarts stalled memory
 275                  * transactions in other address spaces which may cause
 276                  * another fault to occur. If we didn't clear the interrupt at
 277                  * this stage a new IRQ might not be raised when the GPU finds
 278                  * a MMU IRQ is already pending.
 279                  */
 280                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 281                                          KBASE_MMU_FAULT_TYPE_PAGE);
 282
 283                 kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
 284                                           faulting_as->fault_addr >> PAGE_SHIFT,
 285                                           new_pages,
 286                                           op, 1);
 287
 288                 mutex_unlock(&faulting_as->transaction_mutex);
 289                 /* AS transaction end */
 290
 291                 /* reenable this in the mask */
 292                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 293                                          KBASE_MMU_FAULT_TYPE_PAGE);
 294                 kbase_gpu_vm_unlock(kctx);
 295         } else {
 296                 /* failed to extend, handle as a normal PF */
 297                 kbase_gpu_vm_unlock(kctx);
 298                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 299                                 "Page allocation failure");
 300         }
 301
 302 fault_done:
 303         /*
 304          * By this point, the fault was handled in some way,
 305          * so release the ctx refcount
 306          */
 307         kbasep_js_runpool_release_ctx(kbdev, kctx);
 308
 309         atomic_dec(&kbdev->faults_pending);
 310 }
 311
 312 phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx)
 313 {
 314         u64 *page;
 315         int i;
 316         struct page *p;
 317         int new_page_count __maybe_unused;
 318
 319         KBASE_DEBUG_ASSERT(NULL != kctx);
 320         new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages);
 321         kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages);
 322
 323         p = kbase_mem_pool_alloc(&kctx->mem_pool);
 324         if (!p)
 325                 goto sub_pages;
 326
 327 #if defined(CONFIG_MALI_MIPE_ENABLED)
 328         kbase_tlstream_aux_pagesalloc(
 329                         (u32)kctx->id,
 330                         (u64)new_page_count);
 331 #endif
 332
 333         page = kmap(p);
 334         if (NULL == page)
 335                 goto alloc_free;
 336
 337         kbase_process_page_usage_inc(kctx, 1);
 338
 339         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++)
 340                 kctx->kbdev->mmu_mode->entry_invalidate(&page[i]);
 341
 342         kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 343
 344         kunmap(p);
 345         return page_to_phys(p);
 346
 347 alloc_free:
 348         kbase_mem_pool_free(&kctx->mem_pool, p, false);
 349 sub_pages:
 350         kbase_atomic_sub_pages(1, &kctx->used_pages);
 351         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
 352
 353         return 0;
 354 }
 355
 356 KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd);
 357
 358 /* Given PGD PFN for level N, return PGD PFN for level N+1 */
 359 static phys_addr_t mmu_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 360 {
 361         u64 *page;
 362         phys_addr_t target_pgd;
 363         struct page *p;
 364
 365         KBASE_DEBUG_ASSERT(pgd);
 366         KBASE_DEBUG_ASSERT(NULL != kctx);
 367
 368         lockdep_assert_held(&kctx->reg_lock);
 369
 370         /*
 371          * Architecture spec defines level-0 as being the top-most.
 372          * This is a bit unfortunate here, but we keep the same convention.
 373          */
 374         vpfn >>= (3 - level) * 9;
 375         vpfn &= 0x1FF;
 376
 377         p = pfn_to_page(PFN_DOWN(pgd));
 378         page = kmap(p);
 379         if (NULL == page) {
 380                 dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n");
 381                 return 0;
 382         }
 383
 384         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 385
 386         if (!target_pgd) {
 387                 target_pgd = kbase_mmu_alloc_pgd(kctx);
 388                 if (!target_pgd) {
 389                         dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n");
 390                         kunmap(p);
 391                         return 0;
 392                 }
 393
 394                 kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd);
 395
 396                 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 397                 /* Rely on the caller to update the address space flags. */
 398         }
 399
 400         kunmap(p);
 401         return target_pgd;
 402 }
 403
 404 static phys_addr_t mmu_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 405 {
 406         phys_addr_t pgd;
 407         int l;
 408
 409         pgd = kctx->pgd;
 410
 411         for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
 412                 pgd = mmu_get_next_pgd(kctx, pgd, vpfn, l);
 413                 /* Handle failure condition */
 414                 if (!pgd) {
 415                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n");
 416                         return 0;
 417                 }
 418         }
 419
 420         return pgd;
 421 }
 422
 423 static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 424 {
 425         u64 *page;
 426         phys_addr_t target_pgd;
 427
 428         KBASE_DEBUG_ASSERT(pgd);
 429         KBASE_DEBUG_ASSERT(NULL != kctx);
 430
 431         lockdep_assert_held(&kctx->reg_lock);
 432
 433         /*
 434          * Architecture spec defines level-0 as being the top-most.
 435          * This is a bit unfortunate here, but we keep the same convention.
 436          */
 437         vpfn >>= (3 - level) * 9;
 438         vpfn &= 0x1FF;
 439
 440         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 441         /* kmap_atomic should NEVER fail */
 442         KBASE_DEBUG_ASSERT(NULL != page);
 443
 444         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 445         /* As we are recovering from what has already been set up, we should have a target_pgd */
 446         KBASE_DEBUG_ASSERT(0 != target_pgd);
 447         kunmap_atomic(page);
 448         return target_pgd;
 449 }
 450
 451 static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 452 {
 453         phys_addr_t pgd;
 454         int l;
 455
 456         pgd = kctx->pgd;
 457
 458         for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
 459                 pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l);
 460                 /* Should never fail */
 461                 KBASE_DEBUG_ASSERT(0 != pgd);
 462         }
 463
 464         return pgd;
 465 }
 466
 467 static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn,
 468                                               size_t nr)
 469 {
 470         phys_addr_t pgd;
 471         u64 *pgd_page;
 472         struct kbase_mmu_mode const *mmu_mode;
 473
 474         KBASE_DEBUG_ASSERT(NULL != kctx);
 475         KBASE_DEBUG_ASSERT(0 != vpfn);
 476         /* 64-bit address range is the max */
 477         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 478
 479         lockdep_assert_held(&kctx->reg_lock);
 480
 481         mmu_mode = kctx->kbdev->mmu_mode;
 482
 483         while (nr) {
 484                 unsigned int i;
 485                 unsigned int index = vpfn & 0x1FF;
 486                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 487                 struct page *p;
 488
 489                 if (count > nr)
 490                         count = nr;
 491
 492                 pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn);
 493                 KBASE_DEBUG_ASSERT(0 != pgd);
 494
 495                 p = pfn_to_page(PFN_DOWN(pgd));
 496
 497                 pgd_page = kmap_atomic(p);
 498                 KBASE_DEBUG_ASSERT(NULL != pgd_page);
 499
 500                 /* Invalidate the entries we added */
 501                 for (i = 0; i < count; i++)
 502                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
 503
 504                 vpfn += count;
 505                 nr -= count;
 506
 507                 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
 508
 509                 kunmap_atomic(pgd_page);
 510         }
 511 }
 512
 513 /*
 514  * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn'
 515  */
 516 int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
 517                                         phys_addr_t phys, size_t nr,
 518                                         unsigned long flags)
 519 {
 520         phys_addr_t pgd;
 521         u64 *pgd_page;
 522         /* In case the insert_single_page only partially completes we need to be
 523          * able to recover */
 524         bool recover_required = false;
 525         u64 recover_vpfn = vpfn;
 526         size_t recover_count = 0;
 527
 528         KBASE_DEBUG_ASSERT(NULL != kctx);
 529         KBASE_DEBUG_ASSERT(0 != vpfn);
 530         /* 64-bit address range is the max */
 531         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 532
 533         lockdep_assert_held(&kctx->reg_lock);
 534
 535         while (nr) {
 536                 unsigned int i;
 537                 unsigned int index = vpfn & 0x1FF;
 538                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 539                 struct page *p;
 540
 541                 if (count > nr)
 542                         count = nr;
 543
 544                 /*
 545                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 546                  * suboptimal. We don't have to re-parse the whole tree
 547                  * each time (just cache the l0-l2 sequence).
 548                  * On the other hand, it's only a gain when we map more than
 549                  * 256 pages at once (on average). Do we really care?
 550                  */
 551                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 552                 if (!pgd) {
 553                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 554                         if (recover_required) {
 555                                 /* Invalidate the pages we have partially
 556                                  * completed */
 557                                 mmu_insert_pages_failure_recovery(kctx,
 558                                                                   recover_vpfn,
 559                                                                   recover_count);
 560                         }
 561                         return -EINVAL;
 562                 }
 563
 564                 p = pfn_to_page(PFN_DOWN(pgd));
 565                 pgd_page = kmap(p);
 566                 if (!pgd_page) {
 567                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 568                         if (recover_required) {
 569                                 /* Invalidate the pages we have partially
 570                                  * completed */
 571                                 mmu_insert_pages_failure_recovery(kctx,
 572                                                                   recover_vpfn,
 573                                                                   recover_count);
 574                         }
 575                         return -ENOMEM;
 576                 }
 577
 578                 for (i = 0; i < count; i++) {
 579                         unsigned int ofs = index + i;
 580
 581                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 582                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 583                                         phys, flags);
 584                 }
 585
 586                 vpfn += count;
 587                 nr -= count;
 588
 589                 kbase_mmu_sync_pgd(kctx->kbdev,
 590                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 591                                 count * sizeof(u64));
 592
 593                 kunmap(p);
 594                 /* We have started modifying the page table.
 595                  * If further pages need inserting and fail we need to undo what
 596                  * has already taken place */
 597                 recover_required = true;
 598                 recover_count += count;
 599         }
 600         return 0;
 601 }
 602
 603 /*
 604  * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn'
 605  */
 606 int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
 607                                   phys_addr_t *phys, size_t nr,
 608                                   unsigned long flags)
 609 {
 610         phys_addr_t pgd;
 611         u64 *pgd_page;
 612         /* In case the insert_pages only partially completes we need to be able
 613          * to recover */
 614         bool recover_required = false;
 615         u64 recover_vpfn = vpfn;
 616         size_t recover_count = 0;
 617
 618         KBASE_DEBUG_ASSERT(NULL != kctx);
 619         KBASE_DEBUG_ASSERT(0 != vpfn);
 620         /* 64-bit address range is the max */
 621         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 622
 623         lockdep_assert_held(&kctx->reg_lock);
 624
 625         while (nr) {
 626                 unsigned int i;
 627                 unsigned int index = vpfn & 0x1FF;
 628                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 629                 struct page *p;
 630
 631                 if (count > nr)
 632                         count = nr;
 633
 634                 /*
 635                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 636                  * suboptimal. We don't have to re-parse the whole tree
 637                  * each time (just cache the l0-l2 sequence).
 638                  * On the other hand, it's only a gain when we map more than
 639                  * 256 pages at once (on average). Do we really care?
 640                  */
 641                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 642                 if (!pgd) {
 643                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 644                         if (recover_required) {
 645                                 /* Invalidate the pages we have partially
 646                                  * completed */
 647                                 mmu_insert_pages_failure_recovery(kctx,
 648                                                                   recover_vpfn,
 649                                                                   recover_count);
 650                         }
 651                         return -EINVAL;
 652                 }
 653
 654                 p = pfn_to_page(PFN_DOWN(pgd));
 655                 pgd_page = kmap(p);
 656                 if (!pgd_page) {
 657                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 658                         if (recover_required) {
 659                                 /* Invalidate the pages we have partially
 660                                  * completed */
 661                                 mmu_insert_pages_failure_recovery(kctx,
 662                                                                   recover_vpfn,
 663                                                                   recover_count);
 664                         }
 665                         return -ENOMEM;
 666                 }
 667
 668                 for (i = 0; i < count; i++) {
 669                         unsigned int ofs = index + i;
 670
 671                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 672                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 673                                         phys[i], flags);
 674                 }
 675
 676                 phys += count;
 677                 vpfn += count;
 678                 nr -= count;
 679
 680                 kbase_mmu_sync_pgd(kctx->kbdev,
 681                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 682                                 count * sizeof(u64));
 683
 684                 kunmap(p);
 685                 /* We have started modifying the page table. If further pages
 686                  * need inserting and fail we need to undo what has already
 687                  * taken place */
 688                 recover_required = true;
 689                 recover_count += count;
 690         }
 691         return 0;
 692 }
 693
 694 KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
 695
 696 /**
 697  * This function is responsible for validating the MMU PTs
 698  * triggering reguired flushes.
 699  *
 700  * * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 701  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 702  * These locks must be taken in the correct order with respect to others
 703  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 704  * information.
 705  */
 706 static void kbase_mmu_flush(struct kbase_context *kctx, u64 vpfn, size_t nr)
 707 {
 708         struct kbase_device *kbdev;
 709         bool ctx_is_in_runpool;
 710
 711         KBASE_DEBUG_ASSERT(NULL != kctx);
 712
 713         kbdev = kctx->kbdev;
 714
 715         /* We must flush if we're currently running jobs. At the very least, we need to retain the
 716          * context to ensure it doesn't schedule out whilst we're trying to flush it */
 717         ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx);
 718
 719         if (ctx_is_in_runpool) {
 720                 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
 721
 722                 /* Second level check is to try to only do this when jobs are running. The refcount is
 723                  * a heuristic for this. */
 724                 if (kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr].as_busy_refcount >= 2) {
 725                         if (!kbase_pm_context_active_handle_suspend(kbdev,
 726                                 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
 727                                 int ret;
 728                                 u32 op;
 729
 730                                 /* AS transaction begin */
 731                                 mutex_lock(&kbdev->as[
 732                                                 kctx->as_nr].transaction_mutex);
 733
 734                                 if (kbase_hw_has_issue(kbdev,
 735                                                 BASE_HW_ISSUE_6367))
 736                                         op = AS_COMMAND_FLUSH;
 737                                 else
 738                                         op = AS_COMMAND_FLUSH_MEM;
 739
 740                                 ret = kbase_mmu_hw_do_operation(kbdev,
 741                                                         &kbdev->as[kctx->as_nr],
 742                                                         kctx, vpfn, nr,
 743                                                         op, 0);
 744 #if KBASE_GPU_RESET_EN
 745                                 if (ret) {
 746                                         /* Flush failed to complete, assume the
 747                                          * GPU has hung and perform a reset to
 748                                          * recover */
 749                                         dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
 750                                         if (kbase_prepare_to_reset_gpu(kbdev))
 751                                                 kbase_reset_gpu(kbdev);
 752                                 }
 753 #endif /* KBASE_GPU_RESET_EN */
 754
 755                                 mutex_unlock(&kbdev->as[
 756                                                 kctx->as_nr].transaction_mutex);
 757                                 /* AS transaction end */
 758
 759                                 kbase_pm_context_idle(kbdev);
 760                         }
 761                 }
 762                 kbasep_js_runpool_release_ctx(kbdev, kctx);
 763         }
 764 }
 765
 766 /*
 767  * We actually only discard the ATE, and not the page table
 768  * pages. There is a potential DoS here, as we'll leak memory by
 769  * having PTEs that are potentially unused.  Will require physical
 770  * page accounting, so MMU pages are part of the process allocation.
 771  *
 772  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 773  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 774  * These locks must be taken in the correct order with respect to others
 775  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 776  * information.
 777  */
 778 int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
 779 {
 780         phys_addr_t pgd;
 781         u64 *pgd_page;
 782         struct kbase_device *kbdev;
 783         size_t requested_nr = nr;
 784         struct kbase_mmu_mode const *mmu_mode;
 785
 786         KBASE_DEBUG_ASSERT(NULL != kctx);
 787         beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr);
 788
 789         lockdep_assert_held(&kctx->reg_lock);
 790
 791         if (0 == nr) {
 792                 /* early out if nothing to do */
 793                 return 0;
 794         }
 795
 796         kbdev = kctx->kbdev;
 797         mmu_mode = kbdev->mmu_mode;
 798
 799         while (nr) {
 800                 unsigned int i;
 801                 unsigned int index = vpfn & 0x1FF;
 802                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 803                 struct page *p;
 804
 805                 if (count > nr)
 806                         count = nr;
 807
 808                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 809                 if (!pgd) {
 810                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n");
 811                         return -EINVAL;
 812                 }
 813
 814                 p = pfn_to_page(PFN_DOWN(pgd));
 815                 pgd_page = kmap(p);
 816                 if (!pgd_page) {
 817                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n");
 818                         return -ENOMEM;
 819                 }
 820
 821                 for (i = 0; i < count; i++)
 822                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
 823
 824                 vpfn += count;
 825                 nr -= count;
 826
 827                 kbase_mmu_sync_pgd(kctx->kbdev,
 828                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 829                                 count * sizeof(u64));
 830
 831                 kunmap(p);
 832         }
 833
 834         kbase_mmu_flush(kctx, vpfn, requested_nr);
 835         return 0;
 836 }
 837
 838 KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
 839
 840 /**
 841  * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'.
 842  * This call is being triggered as a response to the changes of the mem attributes
 843  *
 844  * @pre : The caller is responsible for validating the memory attributes
 845  *
 846  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 847  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 848  * These locks must be taken in the correct order with respect to others
 849  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 850  * information.
 851  */
 852 int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags)
 853 {
 854         phys_addr_t pgd;
 855         u64 *pgd_page;
 856         size_t requested_nr = nr;
 857         struct kbase_mmu_mode const *mmu_mode;
 858
 859         KBASE_DEBUG_ASSERT(NULL != kctx);
 860         KBASE_DEBUG_ASSERT(0 != vpfn);
 861         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 862
 863         lockdep_assert_held(&kctx->reg_lock);
 864
 865         mmu_mode = kctx->kbdev->mmu_mode;
 866
 867         dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages",
 868                         vpfn, phys, nr);
 869
 870         while (nr) {
 871                 unsigned int i;
 872                 unsigned int index = vpfn & 0x1FF;
 873                 size_t count = KBASE_MMU_PAGE_ENTRIES - index;
 874                 struct page *p;
 875
 876                 if (count > nr)
 877                         count = nr;
 878
 879                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 880                 if (!pgd) {
 881                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n");
 882                         return -EINVAL;
 883                 }
 884
 885                 p = pfn_to_page(PFN_DOWN(pgd));
 886                 pgd_page = kmap(p);
 887                 if (!pgd_page) {
 888                         dev_warn(kctx->kbdev->dev, "kmap failure\n");
 889                         return -ENOMEM;
 890                 }
 891
 892                 for (i = 0; i < count; i++)
 893                         mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i],
 894                                         flags);
 895
 896                 phys += count;
 897                 vpfn += count;
 898                 nr -= count;
 899
 900                 kbase_mmu_sync_pgd(kctx->kbdev,
 901                                 kbase_dma_addr(p) + (index * sizeof(u64)),
 902                                 count * sizeof(u64));
 903
 904                 kunmap(pfn_to_page(PFN_DOWN(pgd)));
 905         }
 906
 907         kbase_mmu_flush(kctx, vpfn, requested_nr);
 908
 909         return 0;
 910 }
 911
 912 /* This is a debug feature only */
 913 static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd)
 914 {
 915         u64 *page;
 916         int i;
 917
 918         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 919         /* kmap_atomic should NEVER fail. */
 920         KBASE_DEBUG_ASSERT(NULL != page);
 921
 922         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
 923                 if (kctx->kbdev->mmu_mode->ate_is_valid(page[i]))
 924                         beenthere(kctx, "live pte %016lx", (unsigned long)page[i]);
 925         }
 926         kunmap_atomic(page);
 927 }
 928
 929 static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer)
 930 {
 931         phys_addr_t target_pgd;
 932         u64 *pgd_page;
 933         int i;
 934         struct kbase_mmu_mode const *mmu_mode;
 935
 936         KBASE_DEBUG_ASSERT(NULL != kctx);
 937         lockdep_assert_held(&kctx->reg_lock);
 938
 939         pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 940         /* kmap_atomic should NEVER fail. */
 941         KBASE_DEBUG_ASSERT(NULL != pgd_page);
 942         /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */
 943         memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE);
 944         kunmap_atomic(pgd_page);
 945         pgd_page = pgd_page_buffer;
 946
 947         mmu_mode = kctx->kbdev->mmu_mode;
 948
 949         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
 950                 target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
 951
 952                 if (target_pgd) {
 953                         if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) {
 954                                 mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64)));
 955                         } else {
 956                                 /*
 957                                  * So target_pte is a level-3 page.
 958                                  * As a leaf, it is safe to free it.
 959                                  * Unless we have live pages attached to it!
 960                                  */
 961                                 mmu_check_unused(kctx, target_pgd);
 962                         }
 963
 964                         beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1);
 965                         if (zap) {
 966                                 struct page *p = phys_to_page(target_pgd);
 967
 968                                 kbase_mem_pool_free(&kctx->mem_pool, p, true);
 969                                 kbase_process_page_usage_dec(kctx, 1);
 970                                 kbase_atomic_sub_pages(1, &kctx->used_pages);
 971                                 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
 972                         }
 973                 }
 974         }
 975 }
 976
 977 int kbase_mmu_init(struct kbase_context *kctx)
 978 {
 979         KBASE_DEBUG_ASSERT(NULL != kctx);
 980         KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages);
 981
 982         /* Preallocate MMU depth of four pages for mmu_teardown_level to use */
 983         kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
 984
 985         if (NULL == kctx->mmu_teardown_pages)
 986                 return -ENOMEM;
 987
 988         return 0;
 989 }
 990
 991 void kbase_mmu_term(struct kbase_context *kctx)
 992 {
 993         KBASE_DEBUG_ASSERT(NULL != kctx);
 994         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
 995
 996         kfree(kctx->mmu_teardown_pages);
 997         kctx->mmu_teardown_pages = NULL;
 998 }
 999
1000 void kbase_mmu_free_pgd(struct kbase_context *kctx)
1001 {
1002         int new_page_count __maybe_unused;
1003
1004         KBASE_DEBUG_ASSERT(NULL != kctx);
1005         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1006
1007         lockdep_assert_held(&kctx->reg_lock);
1008
1009         mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages);
1010
1011         beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd);
1012         kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true);
1013         kbase_process_page_usage_dec(kctx, 1);
1014         new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages);
1015         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1016
1017 #if defined(CONFIG_MALI_MIPE_ENABLED)
1018         kbase_tlstream_aux_pagesalloc(
1019                         (u32)kctx->id,
1020                         (u64)new_page_count);
1021 #endif
1022 }
1023
1024 KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd);
1025
1026 static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left)
1027 {
1028         phys_addr_t target_pgd;
1029         u64 *pgd_page;
1030         int i;
1031         size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64);
1032         size_t dump_size;
1033         struct kbase_mmu_mode const *mmu_mode;
1034
1035         KBASE_DEBUG_ASSERT(NULL != kctx);
1036         lockdep_assert_held(&kctx->reg_lock);
1037
1038         mmu_mode = kctx->kbdev->mmu_mode;
1039
1040         pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd)));
1041         if (!pgd_page) {
1042                 dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n");
1043                 return 0;
1044         }
1045
1046         if (*size_left >= size) {
1047                 /* A modified physical address that contains the page table level */
1048                 u64 m_pgd = pgd | level;
1049
1050                 /* Put the modified physical address in the output buffer */
1051                 memcpy(*buffer, &m_pgd, sizeof(m_pgd));
1052                 *buffer += sizeof(m_pgd);
1053
1054                 /* Followed by the page table itself */
1055                 memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES);
1056                 *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES;
1057
1058                 *size_left -= size;
1059         }
1060
1061         if (level < MIDGARD_MMU_BOTTOMLEVEL) {
1062                 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1063                         if (mmu_mode->pte_is_valid(pgd_page[i])) {
1064                                 target_pgd = mmu_mode->pte_to_phy_addr(
1065                                                 pgd_page[i]);
1066
1067                                 dump_size = kbasep_mmu_dump_level(kctx,
1068                                                 target_pgd, level + 1,
1069                                                 buffer, size_left);
1070                                 if (!dump_size) {
1071                                         kunmap(pfn_to_page(PFN_DOWN(pgd)));
1072                                         return 0;
1073                                 }
1074                                 size += dump_size;
1075                         }
1076                 }
1077         }
1078
1079         kunmap(pfn_to_page(PFN_DOWN(pgd)));
1080
1081         return size;
1082 }
1083
1084 void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages)
1085 {
1086         void *kaddr;
1087         size_t size_left;
1088
1089         KBASE_DEBUG_ASSERT(kctx);
1090
1091         lockdep_assert_held(&kctx->reg_lock);
1092
1093         if (0 == nr_pages) {
1094                 /* can't dump in a 0 sized buffer, early out */
1095                 return NULL;
1096         }
1097
1098         size_left = nr_pages * PAGE_SIZE;
1099
1100         KBASE_DEBUG_ASSERT(0 != size_left);
1101         kaddr = vmalloc_user(size_left);
1102
1103         if (kaddr) {
1104                 u64 end_marker = 0xFFULL;
1105                 char *buffer;
1106                 char *mmu_dump_buffer;
1107                 u64 config[3];
1108                 size_t size;
1109
1110                 buffer = (char *)kaddr;
1111                 mmu_dump_buffer = buffer;
1112
1113                 if (kctx->api_version >= KBASE_API_VERSION(8, 4)) {
1114                         struct kbase_mmu_setup as_setup;
1115
1116                         kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup);
1117                         config[0] = as_setup.transtab;
1118                         config[1] = as_setup.memattr;
1119                         config[2] = 0;
1120                         memcpy(buffer, &config, sizeof(config));
1121                         mmu_dump_buffer += sizeof(config);
1122                         size_left -= sizeof(config);
1123                 }
1124
1125
1126
1127                 size = kbasep_mmu_dump_level(kctx,
1128                                 kctx->pgd,
1129                                 MIDGARD_MMU_TOPLEVEL,
1130                                 &mmu_dump_buffer,
1131                                 &size_left);
1132
1133                 if (!size) {
1134                         vfree(kaddr);
1135                         return NULL;
1136                 }
1137
1138                 /* Add on the size for the end marker */
1139                 size += sizeof(u64);
1140                 /* Add on the size for the config */
1141                 if (kctx->api_version >= KBASE_API_VERSION(8, 4))
1142                         size += sizeof(config);
1143
1144
1145                 if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) {
1146                         /* The buffer isn't big enough - free the memory and return failure */
1147                         vfree(kaddr);
1148                         return NULL;
1149                 }
1150
1151                 /* Add the end marker */
1152                 memcpy(mmu_dump_buffer, &end_marker, sizeof(u64));
1153         }
1154
1155         return kaddr;
1156 }
1157 KBASE_EXPORT_TEST_API(kbase_mmu_dump);
1158
1159 void bus_fault_worker(struct work_struct *data)
1160 {
1161         struct kbase_as *faulting_as;
1162         int as_no;
1163         struct kbase_context *kctx;
1164         struct kbase_device *kbdev;
1165 #if KBASE_GPU_RESET_EN
1166         bool reset_status = false;
1167 #endif /* KBASE_GPU_RESET_EN */
1168
1169         faulting_as = container_of(data, struct kbase_as, work_busfault);
1170
1171         as_no = faulting_as->number;
1172
1173         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
1174
1175         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
1176          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
1177          */
1178         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
1179         if (WARN_ON(!kctx)) {
1180                 atomic_dec(&kbdev->faults_pending);
1181                 return;
1182         }
1183
1184 #if KBASE_GPU_RESET_EN
1185         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1186                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1187                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1188                  * are evicted from the GPU before the switch.
1189                  */
1190                 dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1191                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1192         }
1193 #endif /* KBASE_GPU_RESET_EN */
1194         /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */
1195         if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1196
1197                 /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1198                 /* AS transaction begin */
1199                 mutex_lock(&kbdev->as[as_no].transaction_mutex);
1200
1201                 /* Set the MMU into unmapped mode */
1202                 kbase_mmu_disable_as(kbdev, as_no);
1203
1204                 mutex_unlock(&kbdev->as[as_no].transaction_mutex);
1205                 /* AS transaction end */
1206
1207                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
1208                                          KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1209                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
1210                                          KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1211
1212                 kbase_pm_context_idle(kbdev);
1213         }
1214
1215 #if KBASE_GPU_RESET_EN
1216         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1217                 kbase_reset_gpu(kbdev);
1218 #endif /* KBASE_GPU_RESET_EN */
1219
1220         kbasep_js_runpool_release_ctx(kbdev, kctx);
1221
1222         atomic_dec(&kbdev->faults_pending);
1223 }
1224
1225 const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
1226 {
1227         const char *e;
1228
1229         switch (exception_code) {
1230                 /* Non-Fault Status code */
1231         case 0x00:
1232                 e = "NOT_STARTED/IDLE/OK";
1233                 break;
1234         case 0x01:
1235                 e = "DONE";
1236                 break;
1237         case 0x02:
1238                 e = "INTERRUPTED";
1239                 break;
1240         case 0x03:
1241                 e = "STOPPED";
1242                 break;
1243         case 0x04:
1244                 e = "TERMINATED";
1245                 break;
1246         case 0x08:
1247                 e = "ACTIVE";
1248                 break;
1249                 /* Job exceptions */
1250         case 0x40:
1251                 e = "JOB_CONFIG_FAULT";
1252                 break;
1253         case 0x41:
1254                 e = "JOB_POWER_FAULT";
1255                 break;
1256         case 0x42:
1257                 e = "JOB_READ_FAULT";
1258                 break;
1259         case 0x43:
1260                 e = "JOB_WRITE_FAULT";
1261                 break;
1262         case 0x44:
1263                 e = "JOB_AFFINITY_FAULT";
1264                 break;
1265         case 0x48:
1266                 e = "JOB_BUS_FAULT";
1267                 break;
1268         case 0x50:
1269                 e = "INSTR_INVALID_PC";
1270                 break;
1271         case 0x51:
1272                 e = "INSTR_INVALID_ENC";
1273                 break;
1274         case 0x52:
1275                 e = "INSTR_TYPE_MISMATCH";
1276                 break;
1277         case 0x53:
1278                 e = "INSTR_OPERAND_FAULT";
1279                 break;
1280         case 0x54:
1281                 e = "INSTR_TLS_FAULT";
1282                 break;
1283         case 0x55:
1284                 e = "INSTR_BARRIER_FAULT";
1285                 break;
1286         case 0x56:
1287                 e = "INSTR_ALIGN_FAULT";
1288                 break;
1289         case 0x58:
1290                 e = "DATA_INVALID_FAULT";
1291                 break;
1292         case 0x59:
1293                 e = "TILE_RANGE_FAULT";
1294                 break;
1295         case 0x5A:
1296                 e = "ADDR_RANGE_FAULT";
1297                 break;
1298         case 0x60:
1299                 e = "OUT_OF_MEMORY";
1300                 break;
1301                 /* GPU exceptions */
1302         case 0x80:
1303                 e = "DELAYED_BUS_FAULT";
1304                 break;
1305         case 0x88:
1306                 e = "SHAREABILITY_FAULT";
1307                 break;
1308                 /* MMU exceptions */
1309         case 0xC0:
1310         case 0xC1:
1311         case 0xC2:
1312         case 0xC3:
1313         case 0xC4:
1314         case 0xC5:
1315         case 0xC6:
1316         case 0xC7:
1317                 e = "TRANSLATION_FAULT";
1318                 break;
1319         case 0xC8:
1320                 e = "PERMISSION_FAULT";
1321                 break;
1322         case 0xD0:
1323         case 0xD1:
1324         case 0xD2:
1325         case 0xD3:
1326         case 0xD4:
1327         case 0xD5:
1328         case 0xD6:
1329         case 0xD7:
1330                 e = "TRANSTAB_BUS_FAULT";
1331                 break;
1332         case 0xD8:
1333                 e = "ACCESS_FLAG";
1334                 break;
1335                 break;
1336         default:
1337                 e = "UNKNOWN";
1338                 break;
1339         };
1340
1341         return e;
1342 }
1343
1344 static const char *access_type_name(struct kbase_device *kbdev,
1345                 u32 fault_status)
1346 {
1347         switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1348                 return "UNKNOWN";
1349         case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1350                 return "READ";
1351         case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1352                 return "WRITE";
1353         case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1354                 return "EXECUTE";
1355         default:
1356                 WARN_ON(1);
1357                 return NULL;
1358         }
1359 }
1360
1361 /**
1362  * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on.
1363  */
1364 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
1365                 struct kbase_as *as, const char *reason_str)
1366 {
1367         unsigned long flags;
1368         int exception_type;
1369         int access_type;
1370         int source_id;
1371         int as_no;
1372         struct kbase_device *kbdev;
1373         struct kbasep_js_device_data *js_devdata;
1374
1375 #if KBASE_GPU_RESET_EN
1376         bool reset_status = false;
1377 #endif
1378
1379         as_no = as->number;
1380         kbdev = kctx->kbdev;
1381         js_devdata = &kbdev->js_data;
1382
1383         /* ASSERT that the context won't leave the runpool */
1384         KBASE_DEBUG_ASSERT(kbasep_js_debug_check_ctx_refcount(kbdev, kctx) > 0);
1385
1386         /* decode the fault status */
1387         exception_type = as->fault_status & 0xFF;
1388         access_type = (as->fault_status >> 8) & 0x3;
1389         source_id = (as->fault_status >> 16);
1390
1391         /* terminal fault, print info about the fault */
1392         dev_err(kbdev->dev,
1393                 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
1394                 "Reason: %s\n"
1395                 "raw fault status 0x%X\n"
1396                 "decoded fault status: %s\n"
1397                 "exception type 0x%X: %s\n"
1398                 "access type 0x%X: %s\n"
1399                 "source id 0x%X\n"
1400                 "pid: %d\n",
1401                 as_no, as->fault_addr,
1402                 reason_str,
1403                 as->fault_status,
1404                 (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1405                 exception_type, kbase_exception_name(kbdev, exception_type),
1406                 access_type, access_type_name(kbdev, as->fault_status),
1407                 source_id,
1408                 kctx->pid);
1409
1410         /* hardware counters dump fault handling */
1411         if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) &&
1412                         (kbdev->hwcnt.backend.state ==
1413                                                 KBASE_INSTR_STATE_DUMPING)) {
1414                 unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
1415
1416                 if ((as->fault_addr >= kbdev->hwcnt.addr) &&
1417                                 (as->fault_addr < (kbdev->hwcnt.addr +
1418                                                 (num_core_groups * 2048))))
1419                         kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT;
1420         }
1421
1422         /* Stop the kctx from submitting more jobs and cause it to be scheduled
1423          * out/rescheduled - this will occur on releasing the context's refcount */
1424         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1425         kbasep_js_clear_submit_allowed(js_devdata, kctx);
1426         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1427
1428         /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this
1429          * context can appear in the job slots from this point on */
1430         kbase_backend_jm_kill_jobs_from_kctx(kctx);
1431         /* AS transaction begin */
1432         mutex_lock(&as->transaction_mutex);
1433 #if KBASE_GPU_RESET_EN
1434         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1435                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1436                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1437                  * are evicted from the GPU before the switch.
1438                  */
1439                 dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery.");
1440                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1441         }
1442 #endif /* KBASE_GPU_RESET_EN */
1443         /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1444         kbase_mmu_disable_as(kbdev, as_no);
1445
1446         mutex_unlock(&as->transaction_mutex);
1447         /* AS transaction end */
1448         /* Clear down the fault */
1449         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1450                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1451         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1452                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1453
1454 #if KBASE_GPU_RESET_EN
1455         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1456                 kbase_reset_gpu(kbdev);
1457 #endif /* KBASE_GPU_RESET_EN */
1458 }
1459
1460 void kbasep_as_do_poke(struct work_struct *work)
1461 {
1462         struct kbase_as *as;
1463         struct kbase_device *kbdev;
1464         struct kbase_context *kctx;
1465         unsigned long flags;
1466
1467         KBASE_DEBUG_ASSERT(work);
1468         as = container_of(work, struct kbase_as, poke_work);
1469         kbdev = container_of(as, struct kbase_device, as[as->number]);
1470         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1471
1472         /* GPU power will already be active by virtue of the caller holding a JS
1473          * reference on the address space, and will not release it until this worker
1474          * has finished */
1475
1476         /* Further to the comment above, we know that while this function is running
1477          * the AS will not be released as before the atom is released this workqueue
1478          * is flushed (in kbase_as_poking_timer_release_atom)
1479          */
1480         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number);
1481
1482         /* AS transaction begin */
1483         mutex_lock(&as->transaction_mutex);
1484         /* Force a uTLB invalidate */
1485         kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0,
1486                                   AS_COMMAND_UNLOCK, 0);
1487         mutex_unlock(&as->transaction_mutex);
1488         /* AS transaction end */
1489
1490         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1491         if (as->poke_refcount &&
1492                 !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) {
1493                 /* Only queue up the timer if we need it, and we're not trying to kill it */
1494                 hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL);
1495         }
1496         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1497 }
1498
1499 enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer)
1500 {
1501         struct kbase_as *as;
1502         int queue_work_ret;
1503
1504         KBASE_DEBUG_ASSERT(NULL != timer);
1505         as = container_of(timer, struct kbase_as, poke_timer);
1506         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1507
1508         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1509         KBASE_DEBUG_ASSERT(queue_work_ret);
1510         return HRTIMER_NORESTART;
1511 }
1512
1513 /**
1514  * Retain the poking timer on an atom's context (if the atom hasn't already
1515  * done so), and start the timer (if it's not already started).
1516  *
1517  * This must only be called on a context that's scheduled in, and an atom
1518  * that's running on the GPU.
1519  *
1520  * The caller must hold kbasep_js_device_data::runpool_irq::lock
1521  *
1522  * This can be called safely from atomic context
1523  */
1524 void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1525 {
1526         struct kbase_as *as;
1527
1528         KBASE_DEBUG_ASSERT(kbdev);
1529         KBASE_DEBUG_ASSERT(kctx);
1530         KBASE_DEBUG_ASSERT(katom);
1531         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1532         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1533
1534         if (katom->poking)
1535                 return;
1536
1537         katom->poking = 1;
1538
1539         /* It's safe to work on the as/as_nr without an explicit reference,
1540          * because the caller holds the runpool_irq lock, and the atom itself
1541          * was also running and had already taken a reference  */
1542         as = &kbdev->as[kctx->as_nr];
1543
1544         if (++(as->poke_refcount) == 1) {
1545                 /* First refcount for poke needed: check if not already in flight */
1546                 if (!as->poke_state) {
1547                         /* need to start poking */
1548                         as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT;
1549                         queue_work(as->poke_wq, &as->poke_work);
1550                 }
1551         }
1552 }
1553
1554 /**
1555  * If an atom holds a poking timer, release it and wait for it to finish
1556  *
1557  * This must only be called on a context that's scheduled in, and an atom
1558  * that still has a JS reference on the context
1559  *
1560  * This must \b not be called from atomic context, since it can sleep.
1561  */
1562 void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1563 {
1564         struct kbase_as *as;
1565         unsigned long flags;
1566
1567         KBASE_DEBUG_ASSERT(kbdev);
1568         KBASE_DEBUG_ASSERT(kctx);
1569         KBASE_DEBUG_ASSERT(katom);
1570         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1571
1572         if (!katom->poking)
1573                 return;
1574
1575         as = &kbdev->as[kctx->as_nr];
1576
1577         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1578         KBASE_DEBUG_ASSERT(as->poke_refcount > 0);
1579         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1580
1581         if (--(as->poke_refcount) == 0) {
1582                 as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE;
1583                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1584
1585                 hrtimer_cancel(&as->poke_timer);
1586                 flush_workqueue(as->poke_wq);
1587
1588                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1589
1590                 /* Re-check whether it's still needed */
1591                 if (as->poke_refcount) {
1592                         int queue_work_ret;
1593                         /* Poking still needed:
1594                          * - Another retain will not be starting the timer or queueing work,
1595                          * because it's still marked as in-flight
1596                          * - The hrtimer has finished, and has not started a new timer or
1597                          * queued work because it's been marked as killing
1598                          *
1599                          * So whatever happens now, just queue the work again */
1600                         as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE);
1601                         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1602                         KBASE_DEBUG_ASSERT(queue_work_ret);
1603                 } else {
1604                         /* It isn't - so mark it as not in flight, and not killing */
1605                         as->poke_state = 0u;
1606
1607                         /* The poke associated with the atom has now finished. If this is
1608                          * also the last atom on the context, then we can guarentee no more
1609                          * pokes (and thus no more poking register accesses) will occur on
1610                          * the context until new atoms are run */
1611                 }
1612         }
1613         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1614
1615         katom->poking = 0;
1616 }
1617
1618 void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as)
1619 {
1620         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1621
1622         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1623
1624         if (!kctx) {
1625                 dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n",
1626                                  kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault",
1627                                  as->number, as->fault_addr);
1628
1629                 /* Since no ctx was found, the MMU must be disabled. */
1630                 WARN_ON(as->current_setup.transtab);
1631
1632                 if (kbase_as_has_bus_fault(as)) {
1633                         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1634                                         KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1635                         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1636                                         KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1637                 } else if (kbase_as_has_page_fault(as)) {
1638                         kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1639                                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1640                         kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1641                                         KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1642                 }
1643
1644 #if KBASE_GPU_RESET_EN
1645                 if (kbase_as_has_bus_fault(as) &&
1646                                 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1647                         bool reset_status;
1648                         /*
1649                          * Reset the GPU, like in bus_fault_worker, in case an
1650                          * earlier error hasn't been properly cleared by this
1651                          * point.
1652                          */
1653                         dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1654                         reset_status = kbase_prepare_to_reset_gpu_locked(kbdev);
1655                         if (reset_status)
1656                                 kbase_reset_gpu_locked(kbdev);
1657                 }
1658 #endif /* KBASE_GPU_RESET_EN */
1659
1660                 return;
1661         }
1662
1663         if (kbase_as_has_bus_fault(as)) {
1664                 /*
1665                  * hw counters dumping in progress, signal the
1666                  * other thread that it failed
1667                  */
1668                 if ((kbdev->hwcnt.kctx == kctx) &&
1669                     (kbdev->hwcnt.backend.state ==
1670                                         KBASE_INSTR_STATE_DUMPING))
1671                         kbdev->hwcnt.backend.state =
1672                                                 KBASE_INSTR_STATE_FAULT;
1673
1674                 /*
1675                  * Stop the kctx from submitting more jobs and cause it
1676                  * to be scheduled out/rescheduled when all references
1677                  * to it are released
1678                  */
1679                 kbasep_js_clear_submit_allowed(js_devdata, kctx);
1680
1681                 dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
1682                                 as->number, as->fault_addr);
1683
1684                 /*
1685                  * We need to switch to UNMAPPED mode - but we do this in a
1686                  * worker so that we can sleep
1687                  */
1688                 kbdev->kbase_group_error++;
1689                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault));
1690                 WARN_ON(work_pending(&as->work_busfault));
1691                 queue_work(as->pf_wq, &as->work_busfault);
1692                 atomic_inc(&kbdev->faults_pending);
1693         } else {
1694                 kbdev->kbase_group_error++;
1695                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault));
1696                 WARN_ON(work_pending(&as->work_pagefault));
1697                 queue_work(as->pf_wq, &as->work_pagefault);
1698                 atomic_inc(&kbdev->faults_pending);
1699         }
1700 }
1701
1702 void kbase_flush_mmu_wqs(struct kbase_device *kbdev)
1703 {
1704         int i;
1705
1706         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
1707                 struct kbase_as *as = &kbdev->as[i];
1708
1709                 flush_workqueue(as->pf_wq);
1710         }
1711 }