drivers/gpu/arm/midgard/mali_kbase_mmu.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_mmu.c
  22  * Base kernel MMU management.
  23  */
  24
  25 /* #define DEBUG    1 */
  26 #include <linux/kernel.h>
  27 #include <linux/dma-mapping.h>
  28 #include <mali_kbase.h>
  29 #include <mali_midg_regmap.h>
  30 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  31 #include <mali_kbase_gator.h>
  32 #endif
  33 #if defined(CONFIG_MALI_MIPE_ENABLED)
  34 #include <mali_kbase_tlstream.h>
  35 #endif
  36 #include <mali_kbase_debug.h>
  37
  38 #define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  39
  40 #include <mali_kbase_defs.h>
  41 #include <mali_kbase_hw.h>
  42 #include <mali_kbase_mmu_hw.h>
  43 #include <mali_kbase_hwaccess_jm.h>
  44
  45 #define KBASE_MMU_PAGE_ENTRIES 512
  46
  47 /**
  48  * kbase_mmu_sync_pgd - sync page directory to memory
  49  * @dev:        Device pointer.
  50  * @handle:     Address of DMA region.
  51  * @size:       Size of the region to sync.
  52  *
  53  * This should be called after each page directory update.
  54  */
  55
  56 static void kbase_mmu_sync_pgd(struct device *dev,
  57                 dma_addr_t handle, size_t size)
  58 {
  59         dma_sync_single_for_device(dev, handle, size, DMA_TO_DEVICE);
  60 }
  61
  62 /*
  63  * Definitions:
  64  * - PGD: Page Directory.
  65  * - PTE: Page Table Entry. A 64bit value pointing to the next
  66  *        level of translation
  67  * - ATE: Address Transation Entry. A 64bit value pointing to
  68  *        a 4kB physical page.
  69  */
  70
  71 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
  72                 struct kbase_as *as, const char *reason_str);
  73
  74
  75 static size_t make_multiple(size_t minimum, size_t multiple)
  76 {
  77         size_t remainder = minimum % multiple;
  78
  79         if (remainder == 0)
  80                 return minimum;
  81
  82         return minimum + multiple - remainder;
  83 }
  84
  85 void page_fault_worker(struct work_struct *data)
  86 {
  87         u64 fault_pfn;
  88         u32 fault_status;
  89         size_t new_pages;
  90         size_t fault_rel_pfn;
  91         struct kbase_as *faulting_as;
  92         int as_no;
  93         struct kbase_context *kctx;
  94         struct kbase_device *kbdev;
  95         struct kbase_va_region *region;
  96         int err;
  97         bool grown = false;
  98
  99         faulting_as = container_of(data, struct kbase_as, work_pagefault);
 100         fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
 101         as_no = faulting_as->number;
 102
 103         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
 104
 105         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
 106          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
 107          *
 108          * NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
 109         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
 110
 111         if (kctx == NULL) {
 112                 /* Only handle this if not already suspended */
 113                 if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
 114                         /* Address space has no context, terminate the work */
 115
 116                         /* AS transaction begin */
 117                         mutex_lock(&faulting_as->transaction_mutex);
 118
 119                         kbase_mmu_disable_as(kbdev, as_no);
 120
 121                         mutex_unlock(&faulting_as->transaction_mutex);
 122                         /* AS transaction end */
 123
 124                         kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 125                                         KBASE_MMU_FAULT_TYPE_PAGE);
 126                         kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 127                                         KBASE_MMU_FAULT_TYPE_PAGE);
 128                         kbase_pm_context_idle(kbdev);
 129                 }
 130                 atomic_dec(&kbdev->faults_pending);
 131                 return;
 132         }
 133
 134         KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
 135
 136         fault_status = faulting_as->fault_status;
 137         switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
 138
 139         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
 140                 /* need to check against the region to handle this one */
 141                 break;
 142
 143         case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
 144                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 145                                 "Permission failure");
 146                 goto fault_done;
 147
 148         case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
 149                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 150                                 "Tranlation table bus fault");
 151                 goto fault_done;
 152
 153         case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
 154                 /* nothing to do, but we don't expect this fault currently */
 155                 dev_warn(kbdev->dev, "Access flag unexpectedly set");
 156                 goto fault_done;
 157
 158
 159         default:
 160                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 161                                 "Unknown fault code");
 162                 goto fault_done;
 163         }
 164
 165         /* so we have a translation fault, let's see if it is for growable
 166          * memory */
 167         kbase_gpu_vm_lock(kctx);
 168
 169         region = kbase_region_tracker_find_region_enclosing_address(kctx,
 170                         faulting_as->fault_addr);
 171         if (!region || region->flags & KBASE_REG_FREE) {
 172                 kbase_gpu_vm_unlock(kctx);
 173                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 174                                 "Memory is not mapped on the GPU");
 175                 goto fault_done;
 176         }
 177
 178         if ((region->flags & GROWABLE_FLAGS_REQUIRED)
 179                         != GROWABLE_FLAGS_REQUIRED) {
 180                 kbase_gpu_vm_unlock(kctx);
 181                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 182                                 "Memory is not growable");
 183                 goto fault_done;
 184         }
 185
 186         /* find the size we need to grow it by */
 187         /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
 188          * validating the fault_adress to be within a size_t from the start_pfn */
 189         fault_rel_pfn = fault_pfn - region->start_pfn;
 190
 191         if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
 192                 dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
 193                                 faulting_as->fault_addr, region->start_pfn,
 194                                 region->start_pfn +
 195                                 kbase_reg_current_backed_size(region));
 196
 197                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 198                                 KBASE_MMU_FAULT_TYPE_PAGE);
 199                 /* [1] in case another page fault occurred while we were
 200                  * handling the (duplicate) page fault we need to ensure we
 201                  * don't loose the other page fault as result of us clearing
 202                  * the MMU IRQ. Therefore, after we clear the MMU IRQ we send
 203                  * an UNLOCK command that will retry any stalled memory
 204                  * transaction (which should cause the other page fault to be
 205                  * raised again).
 206                  */
 207                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 208                                 AS_COMMAND_UNLOCK, 1);
 209                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 210                                 KBASE_MMU_FAULT_TYPE_PAGE);
 211                 kbase_gpu_vm_unlock(kctx);
 212
 213                 goto fault_done;
 214         }
 215
 216         new_pages = make_multiple(fault_rel_pfn -
 217                         kbase_reg_current_backed_size(region) + 1,
 218                         region->extent);
 219
 220         /* cap to max vsize */
 221         if (new_pages + kbase_reg_current_backed_size(region) >
 222                         region->nr_pages)
 223                 new_pages = region->nr_pages -
 224                                 kbase_reg_current_backed_size(region);
 225
 226         if (0 == new_pages) {
 227                 /* Duplicate of a fault we've already handled, nothing to do */
 228                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 229                                 KBASE_MMU_FAULT_TYPE_PAGE);
 230                 /* See comment [1] about UNLOCK usage */
 231                 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
 232                                 AS_COMMAND_UNLOCK, 1);
 233                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 234                                 KBASE_MMU_FAULT_TYPE_PAGE);
 235                 kbase_gpu_vm_unlock(kctx);
 236                 goto fault_done;
 237         }
 238
 239         if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
 240                 if (region->gpu_alloc != region->cpu_alloc) {
 241                         if (kbase_alloc_phy_pages_helper(
 242                                         region->cpu_alloc, new_pages) == 0) {
 243                                 grown = true;
 244                         } else {
 245                                 kbase_free_phy_pages_helper(region->gpu_alloc,
 246                                                 new_pages);
 247                         }
 248                 } else {
 249                         grown = true;
 250                 }
 251         }
 252
 253
 254         if (grown) {
 255                 u32 op;
 256
 257                 /* alloc success */
 258                 KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
 259
 260                 /* AS transaction begin */
 261                 mutex_lock(&faulting_as->transaction_mutex);
 262
 263                 /* set up the new pages */
 264                 err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
 265                 if (err) {
 266                         /* failed to insert pages, handle as a normal PF */
 267                         mutex_unlock(&faulting_as->transaction_mutex);
 268                         kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
 269                         if (region->gpu_alloc != region->cpu_alloc)
 270                                 kbase_free_phy_pages_helper(region->cpu_alloc,
 271                                                 new_pages);
 272                         kbase_gpu_vm_unlock(kctx);
 273                         /* The locked VA region will be unlocked and the cache invalidated in here */
 274                         kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 275                                         "Page table update failure");
 276                         goto fault_done;
 277                 }
 278 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 279                 kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
 280 #endif
 281 #if defined(CONFIG_MALI_MIPE_ENABLED)
 282                 kbase_tlstream_aux_pagefault(
 283                                 as_no,
 284                                 atomic_read(&kctx->used_pages));
 285 #endif
 286
 287                 /* flush L2 and unlock the VA (resumes the MMU) */
 288                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
 289                         op = AS_COMMAND_FLUSH;
 290                 else
 291                         op = AS_COMMAND_FLUSH_PT;
 292
 293                 /* clear MMU interrupt - this needs to be done after updating
 294                  * the page tables but before issuing a FLUSH command. The
 295                  * FLUSH cmd has a side effect that it restarts stalled memory
 296                  * transactions in other address spaces which may cause
 297                  * another fault to occur. If we didn't clear the interrupt at
 298                  * this stage a new IRQ might not be raised when the GPU finds
 299                  * a MMU IRQ is already pending.
 300                  */
 301                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
 302                                          KBASE_MMU_FAULT_TYPE_PAGE);
 303
 304                 kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
 305                                           faulting_as->fault_addr >> PAGE_SHIFT,
 306                                           new_pages,
 307                                           op, 1);
 308
 309                 mutex_unlock(&faulting_as->transaction_mutex);
 310                 /* AS transaction end */
 311
 312                 /* reenable this in the mask */
 313                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
 314                                          KBASE_MMU_FAULT_TYPE_PAGE);
 315                 kbase_gpu_vm_unlock(kctx);
 316         } else {
 317                 /* failed to extend, handle as a normal PF */
 318                 kbase_gpu_vm_unlock(kctx);
 319                 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
 320                                 "Page allocation failure");
 321         }
 322
 323 fault_done:
 324         /*
 325          * By this point, the fault was handled in some way,
 326          * so release the ctx refcount
 327          */
 328         kbasep_js_runpool_release_ctx(kbdev, kctx);
 329
 330         atomic_dec(&kbdev->faults_pending);
 331 }
 332
 333 phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx)
 334 {
 335         phys_addr_t pgd;
 336         u64 *page;
 337         int i;
 338         struct page *p;
 339
 340         KBASE_DEBUG_ASSERT(NULL != kctx);
 341         kbase_atomic_add_pages(1, &kctx->used_pages);
 342         kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages);
 343
 344         if (kbase_mem_allocator_alloc(kctx->pgd_allocator, 1, &pgd) != 0)
 345                 goto sub_pages;
 346
 347         p = pfn_to_page(PFN_DOWN(pgd));
 348         page = kmap(p);
 349         if (NULL == page)
 350                 goto alloc_free;
 351
 352         kbase_process_page_usage_inc(kctx, 1);
 353
 354         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++)
 355                 kctx->kbdev->mmu_mode->entry_invalidate(&page[i]);
 356
 357         kbase_mmu_sync_pgd(kctx->kbdev->dev, kbase_dma_addr(p), PAGE_SIZE);
 358
 359         kunmap(pfn_to_page(PFN_DOWN(pgd)));
 360         return pgd;
 361
 362 alloc_free:
 363         kbase_mem_allocator_free(kctx->pgd_allocator, 1, &pgd, false);
 364 sub_pages:
 365         kbase_atomic_sub_pages(1, &kctx->used_pages);
 366         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
 367
 368         return 0;
 369 }
 370
 371 KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd);
 372
 373 /* Given PGD PFN for level N, return PGD PFN for level N+1 */
 374 static phys_addr_t mmu_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 375 {
 376         u64 *page;
 377         phys_addr_t target_pgd;
 378         struct page *p;
 379
 380         KBASE_DEBUG_ASSERT(pgd);
 381         KBASE_DEBUG_ASSERT(NULL != kctx);
 382
 383         lockdep_assert_held(&kctx->reg_lock);
 384
 385         /*
 386          * Architecture spec defines level-0 as being the top-most.
 387          * This is a bit unfortunate here, but we keep the same convention.
 388          */
 389         vpfn >>= (3 - level) * 9;
 390         vpfn &= 0x1FF;
 391
 392         p = pfn_to_page(PFN_DOWN(pgd));
 393         page = kmap(p);
 394         if (NULL == page) {
 395                 dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n");
 396                 return 0;
 397         }
 398
 399         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 400
 401         if (!target_pgd) {
 402                 target_pgd = kbase_mmu_alloc_pgd(kctx);
 403                 if (!target_pgd) {
 404                         dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n");
 405                         kunmap(p);
 406                         return 0;
 407                 }
 408
 409                 kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd);
 410
 411                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 412                                 kbase_dma_addr(p), PAGE_SIZE);
 413                 /* Rely on the caller to update the address space flags. */
 414         }
 415
 416         kunmap(p);
 417         return target_pgd;
 418 }
 419
 420 static phys_addr_t mmu_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 421 {
 422         phys_addr_t pgd;
 423         int l;
 424
 425         pgd = kctx->pgd;
 426
 427         for (l = MIDGARD_MMU_TOPLEVEL; l < 3; l++) {
 428                 pgd = mmu_get_next_pgd(kctx, pgd, vpfn, l);
 429                 /* Handle failure condition */
 430                 if (!pgd) {
 431                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n");
 432                         return 0;
 433                 }
 434         }
 435
 436         return pgd;
 437 }
 438
 439 static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
 440 {
 441         u64 *page;
 442         phys_addr_t target_pgd;
 443
 444         KBASE_DEBUG_ASSERT(pgd);
 445         KBASE_DEBUG_ASSERT(NULL != kctx);
 446
 447         lockdep_assert_held(&kctx->reg_lock);
 448
 449         /*
 450          * Architecture spec defines level-0 as being the top-most.
 451          * This is a bit unfortunate here, but we keep the same convention.
 452          */
 453         vpfn >>= (3 - level) * 9;
 454         vpfn &= 0x1FF;
 455
 456         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 457         /* kmap_atomic should NEVER fail */
 458         KBASE_DEBUG_ASSERT(NULL != page);
 459
 460         target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
 461         /* As we are recovering from what has already been set up, we should have a target_pgd */
 462         KBASE_DEBUG_ASSERT(0 != target_pgd);
 463         kunmap_atomic(page);
 464         return target_pgd;
 465 }
 466
 467 static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
 468 {
 469         phys_addr_t pgd;
 470         int l;
 471
 472         pgd = kctx->pgd;
 473
 474         for (l = MIDGARD_MMU_TOPLEVEL; l < 3; l++) {
 475                 pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l);
 476                 /* Should never fail */
 477                 KBASE_DEBUG_ASSERT(0 != pgd);
 478         }
 479
 480         return pgd;
 481 }
 482
 483 static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn,
 484                                               size_t nr)
 485 {
 486         phys_addr_t pgd;
 487         u64 *pgd_page;
 488         struct kbase_mmu_mode const *mmu_mode;
 489
 490         KBASE_DEBUG_ASSERT(NULL != kctx);
 491         KBASE_DEBUG_ASSERT(0 != vpfn);
 492         /* 64-bit address range is the max */
 493         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 494
 495         lockdep_assert_held(&kctx->reg_lock);
 496
 497         mmu_mode = kctx->kbdev->mmu_mode;
 498
 499         while (nr) {
 500                 unsigned int i;
 501                 unsigned int index = vpfn & 0x1FF;
 502                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 503                 struct page *p;
 504
 505                 if (count > nr)
 506                         count = nr;
 507
 508                 pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn);
 509                 KBASE_DEBUG_ASSERT(0 != pgd);
 510
 511                 p = pfn_to_page(PFN_DOWN(pgd));
 512
 513                 pgd_page = kmap_atomic(p);
 514                 KBASE_DEBUG_ASSERT(NULL != pgd_page);
 515
 516                 /* Invalidate the entries we added */
 517                 for (i = 0; i < count; i++)
 518                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
 519
 520                 vpfn += count;
 521                 nr -= count;
 522
 523                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 524                                            kbase_dma_addr(p),
 525                                            PAGE_SIZE);
 526
 527                 kunmap_atomic(pgd_page);
 528         }
 529 }
 530
 531 /*
 532  * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn'
 533  */
 534 int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
 535                                         phys_addr_t phys, size_t nr,
 536                                         unsigned long flags)
 537 {
 538         phys_addr_t pgd;
 539         u64 *pgd_page;
 540         /* In case the insert_single_page only partially completes we need to be
 541          * able to recover */
 542         bool recover_required = false;
 543         u64 recover_vpfn = vpfn;
 544         size_t recover_count = 0;
 545
 546         KBASE_DEBUG_ASSERT(NULL != kctx);
 547         KBASE_DEBUG_ASSERT(0 != vpfn);
 548         /* 64-bit address range is the max */
 549         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 550
 551         lockdep_assert_held(&kctx->reg_lock);
 552
 553         while (nr) {
 554                 unsigned int i;
 555                 unsigned int index = vpfn & 0x1FF;
 556                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 557                 struct page *p;
 558
 559                 if (count > nr)
 560                         count = nr;
 561
 562                 /*
 563                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 564                  * suboptimal. We don't have to re-parse the whole tree
 565                  * each time (just cache the l0-l2 sequence).
 566                  * On the other hand, it's only a gain when we map more than
 567                  * 256 pages at once (on average). Do we really care?
 568                  */
 569                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 570                 if (!pgd) {
 571                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 572                         if (recover_required) {
 573                                 /* Invalidate the pages we have partially
 574                                  * completed */
 575                                 mmu_insert_pages_failure_recovery(kctx,
 576                                                                   recover_vpfn,
 577                                                                   recover_count);
 578                         }
 579                         return -EINVAL;
 580                 }
 581
 582                 p = pfn_to_page(PFN_DOWN(pgd));
 583                 pgd_page = kmap(p);
 584                 if (!pgd_page) {
 585                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 586                         if (recover_required) {
 587                                 /* Invalidate the pages we have partially
 588                                  * completed */
 589                                 mmu_insert_pages_failure_recovery(kctx,
 590                                                                   recover_vpfn,
 591                                                                   recover_count);
 592                         }
 593                         return -ENOMEM;
 594                 }
 595
 596                 for (i = 0; i < count; i++) {
 597                         unsigned int ofs = index + i;
 598
 599                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 600                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 601                                         phys, flags);
 602                 }
 603
 604                 vpfn += count;
 605                 nr -= count;
 606
 607                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 608                                            kbase_dma_addr(p) +
 609                                            (index * sizeof(u64)),
 610                                            count * sizeof(u64));
 611
 612                 kunmap(p);
 613                 /* We have started modifying the page table.
 614                  * If further pages need inserting and fail we need to undo what
 615                  * has already taken place */
 616                 recover_required = true;
 617                 recover_count += count;
 618         }
 619         return 0;
 620 }
 621
 622 /*
 623  * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn'
 624  */
 625 int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
 626                                   phys_addr_t *phys, size_t nr,
 627                                   unsigned long flags)
 628 {
 629         phys_addr_t pgd;
 630         u64 *pgd_page;
 631         /* In case the insert_pages only partially completes we need to be able
 632          * to recover */
 633         bool recover_required = false;
 634         u64 recover_vpfn = vpfn;
 635         size_t recover_count = 0;
 636
 637         KBASE_DEBUG_ASSERT(NULL != kctx);
 638         KBASE_DEBUG_ASSERT(0 != vpfn);
 639         /* 64-bit address range is the max */
 640         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 641
 642         lockdep_assert_held(&kctx->reg_lock);
 643
 644         while (nr) {
 645                 unsigned int i;
 646                 unsigned int index = vpfn & 0x1FF;
 647                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 648                 struct page *p;
 649
 650                 if (count > nr)
 651                         count = nr;
 652
 653                 /*
 654                  * Repeatedly calling mmu_get_bottom_pte() is clearly
 655                  * suboptimal. We don't have to re-parse the whole tree
 656                  * each time (just cache the l0-l2 sequence).
 657                  * On the other hand, it's only a gain when we map more than
 658                  * 256 pages at once (on average). Do we really care?
 659                  */
 660                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 661                 if (!pgd) {
 662                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
 663                         if (recover_required) {
 664                                 /* Invalidate the pages we have partially
 665                                  * completed */
 666                                 mmu_insert_pages_failure_recovery(kctx,
 667                                                                   recover_vpfn,
 668                                                                   recover_count);
 669                         }
 670                         return -EINVAL;
 671                 }
 672
 673                 p = pfn_to_page(PFN_DOWN(pgd));
 674                 pgd_page = kmap(p);
 675                 if (!pgd_page) {
 676                         dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
 677                         if (recover_required) {
 678                                 /* Invalidate the pages we have partially
 679                                  * completed */
 680                                 mmu_insert_pages_failure_recovery(kctx,
 681                                                                   recover_vpfn,
 682                                                                   recover_count);
 683                         }
 684                         return -ENOMEM;
 685                 }
 686
 687                 for (i = 0; i < count; i++) {
 688                         unsigned int ofs = index + i;
 689
 690                         KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
 691                         kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
 692                                         phys[i], flags);
 693                 }
 694
 695                 phys += count;
 696                 vpfn += count;
 697                 nr -= count;
 698
 699                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 700                                            kbase_dma_addr(p) +
 701                                            (index * sizeof(u64)),
 702                                            count * sizeof(u64));
 703
 704                 kunmap(p);
 705                 /* We have started modifying the page table. If further pages
 706                  * need inserting and fail we need to undo what has already
 707                  * taken place */
 708                 recover_required = true;
 709                 recover_count += count;
 710         }
 711         return 0;
 712 }
 713
 714 KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
 715
 716 /**
 717  * This function is responsible for validating the MMU PTs
 718  * triggering reguired flushes.
 719  *
 720  * * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 721  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 722  * These locks must be taken in the correct order with respect to others
 723  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 724  * information.
 725  */
 726 static void kbase_mmu_flush(struct kbase_context *kctx, u64 vpfn, size_t nr)
 727 {
 728         struct kbase_device *kbdev;
 729         bool ctx_is_in_runpool;
 730
 731         KBASE_DEBUG_ASSERT(NULL != kctx);
 732
 733         kbdev = kctx->kbdev;
 734
 735         /* We must flush if we're currently running jobs. At the very least, we need to retain the
 736          * context to ensure it doesn't schedule out whilst we're trying to flush it */
 737         ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx);
 738
 739         if (ctx_is_in_runpool) {
 740                 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
 741
 742                 /* Second level check is to try to only do this when jobs are running. The refcount is
 743                  * a heuristic for this. */
 744                 if (kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr].as_busy_refcount >= 2) {
 745                         if (!kbase_pm_context_active_handle_suspend(kbdev,
 746                                 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
 747                                 int ret;
 748                                 u32 op;
 749
 750                                 /* AS transaction begin */
 751                                 mutex_lock(&kbdev->as[
 752                                                 kctx->as_nr].transaction_mutex);
 753
 754                                 if (kbase_hw_has_issue(kbdev,
 755                                                 BASE_HW_ISSUE_6367))
 756                                         op = AS_COMMAND_FLUSH;
 757                                 else
 758                                         op = AS_COMMAND_FLUSH_MEM;
 759
 760                                 ret = kbase_mmu_hw_do_operation(kbdev,
 761                                                         &kbdev->as[kctx->as_nr],
 762                                                         kctx, vpfn, nr,
 763                                                         op, 0);
 764 #if KBASE_GPU_RESET_EN
 765                                 if (ret) {
 766                                         /* Flush failed to complete, assume the
 767                                          * GPU has hung and perform a reset to
 768                                          * recover */
 769                                         dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
 770                                         if (kbase_prepare_to_reset_gpu(kbdev))
 771                                                 kbase_reset_gpu(kbdev);
 772                                 }
 773 #endif /* KBASE_GPU_RESET_EN */
 774
 775                                 mutex_unlock(&kbdev->as[
 776                                                 kctx->as_nr].transaction_mutex);
 777                                 /* AS transaction end */
 778
 779                                 kbase_pm_context_idle(kbdev);
 780                         }
 781                 }
 782                 kbasep_js_runpool_release_ctx(kbdev, kctx);
 783         }
 784 }
 785
 786 /*
 787  * We actually only discard the ATE, and not the page table
 788  * pages. There is a potential DoS here, as we'll leak memory by
 789  * having PTEs that are potentially unused.  Will require physical
 790  * page accounting, so MMU pages are part of the process allocation.
 791  *
 792  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 793  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 794  * These locks must be taken in the correct order with respect to others
 795  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 796  * information.
 797  */
 798 int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
 799 {
 800         phys_addr_t pgd;
 801         u64 *pgd_page;
 802         struct kbase_device *kbdev;
 803         size_t requested_nr = nr;
 804         struct kbase_mmu_mode const *mmu_mode;
 805
 806         KBASE_DEBUG_ASSERT(NULL != kctx);
 807         beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr);
 808
 809         lockdep_assert_held(&kctx->reg_lock);
 810
 811         if (0 == nr) {
 812                 /* early out if nothing to do */
 813                 return 0;
 814         }
 815
 816         kbdev = kctx->kbdev;
 817         mmu_mode = kbdev->mmu_mode;
 818
 819         while (nr) {
 820                 unsigned int i;
 821                 unsigned int index = vpfn & 0x1FF;
 822                 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
 823                 struct page *p;
 824
 825                 if (count > nr)
 826                         count = nr;
 827
 828                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 829                 if (!pgd) {
 830                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n");
 831                         return -EINVAL;
 832                 }
 833
 834                 p = pfn_to_page(PFN_DOWN(pgd));
 835                 pgd_page = kmap(p);
 836                 if (!pgd_page) {
 837                         dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n");
 838                         return -ENOMEM;
 839                 }
 840
 841                 for (i = 0; i < count; i++)
 842                         mmu_mode->entry_invalidate(&pgd_page[index + i]);
 843
 844                 vpfn += count;
 845                 nr -= count;
 846
 847                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 848                                            kbase_dma_addr(p) +
 849                                            (index * sizeof(u64)),
 850                                            count * sizeof(u64));
 851
 852                 kunmap(p);
 853         }
 854
 855         kbase_mmu_flush(kctx, vpfn, requested_nr);
 856         return 0;
 857 }
 858
 859 KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
 860
 861 /**
 862  * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'.
 863  * This call is being triggered as a response to the changes of the mem attributes
 864  *
 865  * @pre : The caller is responsible for validating the memory attributes
 866  *
 867  * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
 868  * currently scheduled into the runpool, and so potentially uses a lot of locks.
 869  * These locks must be taken in the correct order with respect to others
 870  * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
 871  * information.
 872  */
 873 int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags)
 874 {
 875         phys_addr_t pgd;
 876         u64 *pgd_page;
 877         size_t requested_nr = nr;
 878         struct kbase_mmu_mode const *mmu_mode;
 879
 880         KBASE_DEBUG_ASSERT(NULL != kctx);
 881         KBASE_DEBUG_ASSERT(0 != vpfn);
 882         KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
 883
 884         lockdep_assert_held(&kctx->reg_lock);
 885
 886         mmu_mode = kctx->kbdev->mmu_mode;
 887
 888         dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages",
 889                         vpfn, phys, nr);
 890
 891         while (nr) {
 892                 unsigned int i;
 893                 unsigned int index = vpfn & 0x1FF;
 894                 size_t count = KBASE_MMU_PAGE_ENTRIES - index;
 895                 struct page *p;
 896
 897                 if (count > nr)
 898                         count = nr;
 899
 900                 pgd = mmu_get_bottom_pgd(kctx, vpfn);
 901                 if (!pgd) {
 902                         dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n");
 903                         return -EINVAL;
 904                 }
 905
 906                 p = pfn_to_page(PFN_DOWN(pgd));
 907                 pgd_page = kmap(p);
 908                 if (!pgd_page) {
 909                         dev_warn(kctx->kbdev->dev, "kmap failure\n");
 910                         return -ENOMEM;
 911                 }
 912
 913                 for (i = 0; i < count; i++)
 914                         mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i],
 915                                         flags);
 916
 917                 phys += count;
 918                 vpfn += count;
 919                 nr -= count;
 920
 921                 kbase_mmu_sync_pgd(kctx->kbdev->dev,
 922                                            kbase_dma_addr(p) +
 923                                            (index * sizeof(u64)),
 924                                            count * sizeof(u64));
 925
 926                 kunmap(pfn_to_page(PFN_DOWN(pgd)));
 927         }
 928
 929         kbase_mmu_flush(kctx, vpfn, requested_nr);
 930
 931         return 0;
 932 }
 933
 934 /* This is a debug feature only */
 935 static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd)
 936 {
 937         u64 *page;
 938         int i;
 939
 940         page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 941         /* kmap_atomic should NEVER fail. */
 942         KBASE_DEBUG_ASSERT(NULL != page);
 943
 944         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
 945                 if (kctx->kbdev->mmu_mode->ate_is_valid(page[i]))
 946                         beenthere(kctx, "live pte %016lx", (unsigned long)page[i]);
 947         }
 948         kunmap_atomic(page);
 949 }
 950
 951 static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer)
 952 {
 953         phys_addr_t target_pgd;
 954         u64 *pgd_page;
 955         int i;
 956         struct kbase_mmu_mode const *mmu_mode;
 957
 958         KBASE_DEBUG_ASSERT(NULL != kctx);
 959         lockdep_assert_held(&kctx->reg_lock);
 960
 961         pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
 962         /* kmap_atomic should NEVER fail. */
 963         KBASE_DEBUG_ASSERT(NULL != pgd_page);
 964         /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */
 965         memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE);
 966         kunmap_atomic(pgd_page);
 967         pgd_page = pgd_page_buffer;
 968
 969         mmu_mode = kctx->kbdev->mmu_mode;
 970
 971         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
 972                 target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
 973
 974                 if (target_pgd) {
 975                         if (level < 2) {
 976                                 mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64)));
 977                         } else {
 978                                 /*
 979                                  * So target_pte is a level-3 page.
 980                                  * As a leaf, it is safe to free it.
 981                                  * Unless we have live pages attached to it!
 982                                  */
 983                                 mmu_check_unused(kctx, target_pgd);
 984                         }
 985
 986                         beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1);
 987                         if (zap) {
 988                                 kbase_mem_allocator_free(kctx->pgd_allocator, 1, &target_pgd, true);
 989                                 kbase_process_page_usage_dec(kctx, 1);
 990                                 kbase_atomic_sub_pages(1, &kctx->used_pages);
 991                                 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
 992                         }
 993                 }
 994         }
 995 }
 996
 997 int kbase_mmu_init(struct kbase_context *kctx)
 998 {
 999         KBASE_DEBUG_ASSERT(NULL != kctx);
1000         KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages);
1001
1002         /* Preallocate MMU depth of four pages for mmu_teardown_level to use */
1003         kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
1004
1005         if (NULL == kctx->mmu_teardown_pages)
1006                 return -ENOMEM;
1007
1008         return 0;
1009 }
1010
1011 void kbase_mmu_term(struct kbase_context *kctx)
1012 {
1013         KBASE_DEBUG_ASSERT(NULL != kctx);
1014         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1015
1016         kfree(kctx->mmu_teardown_pages);
1017         kctx->mmu_teardown_pages = NULL;
1018 }
1019
1020 void kbase_mmu_free_pgd(struct kbase_context *kctx)
1021 {
1022         KBASE_DEBUG_ASSERT(NULL != kctx);
1023         KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1024
1025         lockdep_assert_held(&kctx->reg_lock);
1026
1027         mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages);
1028
1029         beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd);
1030         kbase_mem_allocator_free(kctx->pgd_allocator, 1, &kctx->pgd, true);
1031         kbase_process_page_usage_dec(kctx, 1);
1032         kbase_atomic_sub_pages(1, &kctx->used_pages);
1033         kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1034 }
1035
1036 KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd);
1037
1038 static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left)
1039 {
1040         phys_addr_t target_pgd;
1041         u64 *pgd_page;
1042         int i;
1043         size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64);
1044         size_t dump_size;
1045         struct kbase_mmu_mode const *mmu_mode;
1046
1047         KBASE_DEBUG_ASSERT(NULL != kctx);
1048         lockdep_assert_held(&kctx->reg_lock);
1049
1050         mmu_mode = kctx->kbdev->mmu_mode;
1051
1052         pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd)));
1053         if (!pgd_page) {
1054                 dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n");
1055                 return 0;
1056         }
1057
1058         if (*size_left >= size) {
1059                 /* A modified physical address that contains the page table level */
1060                 u64 m_pgd = pgd | level;
1061
1062                 /* Put the modified physical address in the output buffer */
1063                 memcpy(*buffer, &m_pgd, sizeof(m_pgd));
1064                 *buffer += sizeof(m_pgd);
1065
1066                 /* Followed by the page table itself */
1067                 memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES);
1068                 *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES;
1069
1070                 *size_left -= size;
1071         }
1072
1073         for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1074                 if (mmu_mode->pte_is_valid(pgd_page[i])) {
1075                         target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
1076
1077                         dump_size = kbasep_mmu_dump_level(kctx, target_pgd, level + 1, buffer, size_left);
1078                         if (!dump_size) {
1079                                 kunmap(pfn_to_page(PFN_DOWN(pgd)));
1080                                 return 0;
1081                         }
1082                         size += dump_size;
1083                 }
1084         }
1085
1086         kunmap(pfn_to_page(PFN_DOWN(pgd)));
1087
1088         return size;
1089 }
1090
1091 void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages)
1092 {
1093         void *kaddr;
1094         size_t size_left;
1095
1096         KBASE_DEBUG_ASSERT(kctx);
1097
1098         lockdep_assert_held(&kctx->reg_lock);
1099
1100         if (0 == nr_pages) {
1101                 /* can't find in a 0 sized buffer, early out */
1102                 return NULL;
1103         }
1104
1105         size_left = nr_pages * PAGE_SIZE;
1106
1107         KBASE_DEBUG_ASSERT(0 != size_left);
1108         kaddr = vmalloc_user(size_left);
1109
1110         if (kaddr) {
1111                 u64 end_marker = 0xFFULL;
1112                 char *buffer = (char *)kaddr;
1113                 size_t size = kbasep_mmu_dump_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, &buffer, &size_left);
1114
1115                 if (!size) {
1116                         vfree(kaddr);
1117                         return NULL;
1118                 }
1119
1120                 /* Add on the size for the end marker */
1121                 size += sizeof(u64);
1122
1123                 if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) {
1124                         /* The buffer isn't big enough - free the memory and return failure */
1125                         vfree(kaddr);
1126                         return NULL;
1127                 }
1128
1129                 /* Add the end marker */
1130                 memcpy(buffer, &end_marker, sizeof(u64));
1131         }
1132
1133         return kaddr;
1134 }
1135 KBASE_EXPORT_TEST_API(kbase_mmu_dump);
1136
1137 void bus_fault_worker(struct work_struct *data)
1138 {
1139         struct kbase_as *faulting_as;
1140         int as_no;
1141         struct kbase_context *kctx;
1142         struct kbase_device *kbdev;
1143 #if KBASE_GPU_RESET_EN
1144         bool reset_status = false;
1145 #endif /* KBASE_GPU_RESET_EN */
1146
1147         faulting_as = container_of(data, struct kbase_as, work_busfault);
1148
1149         as_no = faulting_as->number;
1150
1151         kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
1152
1153         /* Grab the context that was already refcounted in kbase_mmu_interrupt().
1154          * Therefore, it cannot be scheduled out of this AS until we explicitly release it
1155          *
1156          * NOTE: NULL can be returned here if we're gracefully handling a spurious interrupt */
1157         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
1158 #if KBASE_GPU_RESET_EN
1159         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1160                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1161                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1162                  * are evicted from the GPU before the switch.
1163                  */
1164                 dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1165                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1166         }
1167 #endif /* KBASE_GPU_RESET_EN */
1168         /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */
1169         if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1170
1171                 /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1172                 /* AS transaction begin */
1173                 mutex_lock(&kbdev->as[as_no].transaction_mutex);
1174
1175                 /* Set the MMU into unmapped mode */
1176                 kbase_mmu_disable_as(kbdev, as_no);
1177
1178                 mutex_unlock(&kbdev->as[as_no].transaction_mutex);
1179                 /* AS transaction end */
1180
1181                 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
1182                                          KBASE_MMU_FAULT_TYPE_BUS);
1183                 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
1184                                          KBASE_MMU_FAULT_TYPE_BUS);
1185
1186                 kbase_pm_context_idle(kbdev);
1187         }
1188 #if KBASE_GPU_RESET_EN
1189         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1190                 kbase_reset_gpu(kbdev);
1191 #endif /* KBASE_GPU_RESET_EN */
1192         /* By this point, the fault was handled in some way, so release the ctx refcount */
1193         if (kctx != NULL)
1194                 kbasep_js_runpool_release_ctx(kbdev, kctx);
1195
1196         atomic_dec(&kbdev->faults_pending);
1197 }
1198
1199 const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
1200 {
1201         const char *e;
1202
1203         switch (exception_code) {
1204                 /* Non-Fault Status code */
1205         case 0x00:
1206                 e = "NOT_STARTED/IDLE/OK";
1207                 break;
1208         case 0x01:
1209                 e = "DONE";
1210                 break;
1211         case 0x02:
1212                 e = "INTERRUPTED";
1213                 break;
1214         case 0x03:
1215                 e = "STOPPED";
1216                 break;
1217         case 0x04:
1218                 e = "TERMINATED";
1219                 break;
1220         case 0x08:
1221                 e = "ACTIVE";
1222                 break;
1223                 /* Job exceptions */
1224         case 0x40:
1225                 e = "JOB_CONFIG_FAULT";
1226                 break;
1227         case 0x41:
1228                 e = "JOB_POWER_FAULT";
1229                 break;
1230         case 0x42:
1231                 e = "JOB_READ_FAULT";
1232                 break;
1233         case 0x43:
1234                 e = "JOB_WRITE_FAULT";
1235                 break;
1236         case 0x44:
1237                 e = "JOB_AFFINITY_FAULT";
1238                 break;
1239         case 0x48:
1240                 e = "JOB_BUS_FAULT";
1241                 break;
1242         case 0x50:
1243                 e = "INSTR_INVALID_PC";
1244                 break;
1245         case 0x51:
1246                 e = "INSTR_INVALID_ENC";
1247                 break;
1248         case 0x52:
1249                 e = "INSTR_TYPE_MISMATCH";
1250                 break;
1251         case 0x53:
1252                 e = "INSTR_OPERAND_FAULT";
1253                 break;
1254         case 0x54:
1255                 e = "INSTR_TLS_FAULT";
1256                 break;
1257         case 0x55:
1258                 e = "INSTR_BARRIER_FAULT";
1259                 break;
1260         case 0x56:
1261                 e = "INSTR_ALIGN_FAULT";
1262                 break;
1263         case 0x58:
1264                 e = "DATA_INVALID_FAULT";
1265                 break;
1266         case 0x59:
1267                 e = "TILE_RANGE_FAULT";
1268                 break;
1269         case 0x5A:
1270                 e = "ADDR_RANGE_FAULT";
1271                 break;
1272         case 0x60:
1273                 e = "OUT_OF_MEMORY";
1274                 break;
1275                 /* GPU exceptions */
1276         case 0x80:
1277                 e = "DELAYED_BUS_FAULT";
1278                 break;
1279         case 0x88:
1280                 e = "SHAREABILITY_FAULT";
1281                 break;
1282                 /* MMU exceptions */
1283         case 0xC0:
1284         case 0xC1:
1285         case 0xC2:
1286         case 0xC3:
1287         case 0xC4:
1288         case 0xC5:
1289         case 0xC6:
1290         case 0xC7:
1291                 e = "TRANSLATION_FAULT";
1292                 break;
1293         case 0xC8:
1294                 e = "PERMISSION_FAULT";
1295                 break;
1296         case 0xD0:
1297         case 0xD1:
1298         case 0xD2:
1299         case 0xD3:
1300         case 0xD4:
1301         case 0xD5:
1302         case 0xD6:
1303         case 0xD7:
1304                 e = "TRANSTAB_BUS_FAULT";
1305                 break;
1306         case 0xD8:
1307                 e = "ACCESS_FLAG";
1308                 break;
1309                 break;
1310         default:
1311                 e = "UNKNOWN";
1312                 break;
1313         };
1314
1315         return e;
1316 }
1317
1318 static const char *access_type_name(struct kbase_device *kbdev,
1319                 u32 fault_status)
1320 {
1321         switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1322                 return "UNKNOWN";
1323         case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1324                 return "READ";
1325         case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1326                 return "WRITE";
1327         case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1328                 return "EXECUTE";
1329         default:
1330                 KBASE_DEBUG_ASSERT(0);
1331                 return NULL;
1332         }
1333 }
1334
1335 /**
1336  * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on.
1337  */
1338 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
1339                 struct kbase_as *as, const char *reason_str)
1340 {
1341         unsigned long flags;
1342         int exception_type;
1343         int access_type;
1344         int source_id;
1345         int as_no;
1346         struct kbase_device *kbdev;
1347         struct kbasep_js_device_data *js_devdata;
1348
1349 #if KBASE_GPU_RESET_EN
1350         bool reset_status = false;
1351 #endif
1352
1353         as_no = as->number;
1354         kbdev = kctx->kbdev;
1355         js_devdata = &kbdev->js_data;
1356
1357         /* ASSERT that the context won't leave the runpool */
1358         KBASE_DEBUG_ASSERT(kbasep_js_debug_check_ctx_refcount(kbdev, kctx) > 0);
1359
1360         /* decode the fault status */
1361         exception_type = as->fault_status & 0xFF;
1362         access_type = (as->fault_status >> 8) & 0x3;
1363         source_id = (as->fault_status >> 16);
1364
1365         /* terminal fault, print info about the fault */
1366         dev_err(kbdev->dev,
1367                 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
1368                 "Reason: %s\n"
1369                 "raw fault status 0x%X\n"
1370                 "decoded fault status: %s\n"
1371                 "exception type 0x%X: %s\n"
1372                 "access type 0x%X: %s\n"
1373                 "source id 0x%X\n"
1374                 "pid: %d\n",
1375                 as_no, as->fault_addr,
1376                 reason_str,
1377                 as->fault_status,
1378                 (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1379                 exception_type, kbase_exception_name(kbdev, exception_type),
1380                 access_type, access_type_name(kbdev, as->fault_status),
1381                 source_id,
1382                 kctx->pid);
1383
1384         /* hardware counters dump fault handling */
1385         if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) &&
1386                         (kbdev->hwcnt.backend.state ==
1387                                                 KBASE_INSTR_STATE_DUMPING)) {
1388                 unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
1389
1390                 if ((as->fault_addr >= kbdev->hwcnt.addr) &&
1391                                 (as->fault_addr < (kbdev->hwcnt.addr +
1392                                                 (num_core_groups * 2048))))
1393                         kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT;
1394         }
1395
1396         /* Stop the kctx from submitting more jobs and cause it to be scheduled
1397          * out/rescheduled - this will occur on releasing the context's refcount */
1398         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1399         kbasep_js_clear_submit_allowed(js_devdata, kctx);
1400         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1401
1402         /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this
1403          * context can appear in the job slots from this point on */
1404         kbase_backend_jm_kill_jobs_from_kctx(kctx);
1405         /* AS transaction begin */
1406         mutex_lock(&as->transaction_mutex);
1407 #if KBASE_GPU_RESET_EN
1408         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1409                 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1410                  * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1411                  * are evicted from the GPU before the switch.
1412                  */
1413                 dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery.");
1414                 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1415         }
1416 #endif /* KBASE_GPU_RESET_EN */
1417         /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1418         kbase_mmu_disable_as(kbdev, as_no);
1419
1420         mutex_unlock(&as->transaction_mutex);
1421         /* AS transaction end */
1422         /* Clear down the fault */
1423         kbase_mmu_hw_clear_fault(kbdev, as, kctx, KBASE_MMU_FAULT_TYPE_PAGE);
1424         kbase_mmu_hw_enable_fault(kbdev, as, kctx, KBASE_MMU_FAULT_TYPE_PAGE);
1425
1426 #if KBASE_GPU_RESET_EN
1427         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1428                 kbase_reset_gpu(kbdev);
1429 #endif /* KBASE_GPU_RESET_EN */
1430 }
1431
1432 void kbasep_as_do_poke(struct work_struct *work)
1433 {
1434         struct kbase_as *as;
1435         struct kbase_device *kbdev;
1436         struct kbase_context *kctx;
1437         unsigned long flags;
1438
1439         KBASE_DEBUG_ASSERT(work);
1440         as = container_of(work, struct kbase_as, poke_work);
1441         kbdev = container_of(as, struct kbase_device, as[as->number]);
1442         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1443
1444         /* GPU power will already be active by virtue of the caller holding a JS
1445          * reference on the address space, and will not release it until this worker
1446          * has finished */
1447
1448         /* Further to the comment above, we know that while this function is running
1449          * the AS will not be released as before the atom is released this workqueue
1450          * is flushed (in kbase_as_poking_timer_release_atom)
1451          */
1452         kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number);
1453
1454         /* AS transaction begin */
1455         mutex_lock(&as->transaction_mutex);
1456         /* Force a uTLB invalidate */
1457         kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0,
1458                                   AS_COMMAND_UNLOCK, 0);
1459         mutex_unlock(&as->transaction_mutex);
1460         /* AS transaction end */
1461
1462         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1463         if (as->poke_refcount &&
1464                 !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) {
1465                 /* Only queue up the timer if we need it, and we're not trying to kill it */
1466                 hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL);
1467         }
1468         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1469 }
1470
1471 enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer)
1472 {
1473         struct kbase_as *as;
1474         int queue_work_ret;
1475
1476         KBASE_DEBUG_ASSERT(NULL != timer);
1477         as = container_of(timer, struct kbase_as, poke_timer);
1478         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1479
1480         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1481         KBASE_DEBUG_ASSERT(queue_work_ret);
1482         return HRTIMER_NORESTART;
1483 }
1484
1485 /**
1486  * Retain the poking timer on an atom's context (if the atom hasn't already
1487  * done so), and start the timer (if it's not already started).
1488  *
1489  * This must only be called on a context that's scheduled in, and an atom
1490  * that's running on the GPU.
1491  *
1492  * The caller must hold kbasep_js_device_data::runpool_irq::lock
1493  *
1494  * This can be called safely from atomic context
1495  */
1496 void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1497 {
1498         struct kbase_as *as;
1499
1500         KBASE_DEBUG_ASSERT(kbdev);
1501         KBASE_DEBUG_ASSERT(kctx);
1502         KBASE_DEBUG_ASSERT(katom);
1503         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1504         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1505
1506         if (katom->poking)
1507                 return;
1508
1509         katom->poking = 1;
1510
1511         /* It's safe to work on the as/as_nr without an explicit reference,
1512          * because the caller holds the runpool_irq lock, and the atom itself
1513          * was also running and had already taken a reference  */
1514         as = &kbdev->as[kctx->as_nr];
1515
1516         if (++(as->poke_refcount) == 1) {
1517                 /* First refcount for poke needed: check if not already in flight */
1518                 if (!as->poke_state) {
1519                         /* need to start poking */
1520                         as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT;
1521                         queue_work(as->poke_wq, &as->poke_work);
1522                 }
1523         }
1524 }
1525
1526 /**
1527  * If an atom holds a poking timer, release it and wait for it to finish
1528  *
1529  * This must only be called on a context that's scheduled in, and an atom
1530  * that still has a JS reference on the context
1531  *
1532  * This must \b not be called from atomic context, since it can sleep.
1533  */
1534 void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1535 {
1536         struct kbase_as *as;
1537         unsigned long flags;
1538
1539         KBASE_DEBUG_ASSERT(kbdev);
1540         KBASE_DEBUG_ASSERT(kctx);
1541         KBASE_DEBUG_ASSERT(katom);
1542         KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1543
1544         if (!katom->poking)
1545                 return;
1546
1547         as = &kbdev->as[kctx->as_nr];
1548
1549         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1550         KBASE_DEBUG_ASSERT(as->poke_refcount > 0);
1551         KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1552
1553         if (--(as->poke_refcount) == 0) {
1554                 as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE;
1555                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1556
1557                 hrtimer_cancel(&as->poke_timer);
1558                 flush_workqueue(as->poke_wq);
1559
1560                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
1561
1562                 /* Re-check whether it's still needed */
1563                 if (as->poke_refcount) {
1564                         int queue_work_ret;
1565                         /* Poking still needed:
1566                          * - Another retain will not be starting the timer or queueing work,
1567                          * because it's still marked as in-flight
1568                          * - The hrtimer has finished, and has not started a new timer or
1569                          * queued work because it's been marked as killing
1570                          *
1571                          * So whatever happens now, just queue the work again */
1572                         as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE);
1573                         queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1574                         KBASE_DEBUG_ASSERT(queue_work_ret);
1575                 } else {
1576                         /* It isn't - so mark it as not in flight, and not killing */
1577                         as->poke_state = 0u;
1578
1579                         /* The poke associated with the atom has now finished. If this is
1580                          * also the last atom on the context, then we can guarentee no more
1581                          * pokes (and thus no more poking register accesses) will occur on
1582                          * the context until new atoms are run */
1583                 }
1584         }
1585         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
1586
1587         katom->poking = 0;
1588 }
1589
1590 void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as)
1591 {
1592         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1593
1594         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1595
1596         if (kctx == NULL) {
1597                 dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n",
1598                                  kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault",
1599                                  as->number, as->fault_addr);
1600         }
1601
1602         if (kbase_as_has_bus_fault(as)) {
1603                 if (kctx) {
1604                         /*
1605                          * hw counters dumping in progress, signal the
1606                          * other thread that it failed
1607                          */
1608                         if ((kbdev->hwcnt.kctx == kctx) &&
1609                             (kbdev->hwcnt.backend.state ==
1610                                                 KBASE_INSTR_STATE_DUMPING))
1611                                 kbdev->hwcnt.backend.state =
1612                                                         KBASE_INSTR_STATE_FAULT;
1613
1614                         /*
1615                          * Stop the kctx from submitting more jobs and cause it
1616                          * to be scheduled out/rescheduled when all references
1617                          * to it are released
1618                          */
1619                         kbasep_js_clear_submit_allowed(js_devdata, kctx);
1620
1621                         dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
1622                                         as->number, as->fault_addr);
1623
1624                 }
1625
1626                 /*
1627                  * We need to switch to UNMAPPED mode - but we do this in a
1628                  * worker so that we can sleep
1629                  */
1630                 kbdev->kbase_group_error++;
1631                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault));
1632                 WARN_ON(work_pending(&as->work_busfault));
1633                 INIT_WORK(&as->work_busfault, bus_fault_worker);
1634                 queue_work(as->pf_wq, &as->work_busfault);
1635                 atomic_inc(&kbdev->faults_pending);
1636         } else {
1637                 kbdev->kbase_group_error++;
1638                 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault));
1639                 WARN_ON(work_pending(&as->work_pagefault));
1640                 INIT_WORK(&as->work_pagefault, page_fault_worker);
1641                 queue_work(as->pf_wq, &as->work_pagefault);
1642                 atomic_inc(&kbdev->faults_pending);
1643         }
1644 }
1645
1646 void kbase_flush_mmu_wqs(struct kbase_device *kbdev)
1647 {
1648         int i;
1649
1650         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
1651                 struct kbase_as *as = &kbdev->as[i];
1652
1653                 flush_workqueue(as->pf_wq);
1654         }
1655 }