drivers/gpu/arm/midgard/mali_kbase_jd.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 #if defined(CONFIG_DMA_SHARED_BUFFER)
  21 #include <linux/dma-buf.h>
  22 #endif                          /* defined(CONFIG_DMA_SHARED_BUFFER) */
  23 #ifdef CONFIG_COMPAT
  24 #include <linux/compat.h>
  25 #endif
  26 #include <mali_kbase.h>
  27 #include <mali_kbase_uku.h>
  28 #ifdef CONFIG_UMP
  29 #include <linux/ump.h>
  30 #endif                          /* CONFIG_UMP */
  31 #include <linux/random.h>
  32 #include <linux/version.h>
  33 #include <linux/ratelimit.h>
  34 #include <linux/pagemap.h>
  35
  36 #include <mali_kbase_jm.h>
  37 #include <mali_kbase_hwaccess_jm.h>
  38
  39 #if defined(CONFIG_MALI_MIPE_ENABLED)
  40 #include <mali_kbase_tlstream.h>
  41 #endif
  42
  43 #define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  44
  45 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
  46 /* random32 was renamed to prandom_u32 in 3.8 */
  47 #define prandom_u32 random32
  48 #endif
  49
  50 /* Return whether katom will run on the GPU or not. Currently only soft jobs and
  51  * dependency-only atoms do not run on the GPU */
  52 #define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) ||  \
  53                         ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) ==    \
  54                                                         BASE_JD_REQ_DEP)))
  55 /*
  56  * This is the kernel side of the API. Only entry points are:
  57  * - kbase_jd_submit(): Called from userspace to submit a single bag
  58  * - kbase_jd_done(): Called from interrupt context to track the
  59  *   completion of a job.
  60  * Callouts:
  61  * - to the job manager (enqueue a job)
  62  * - to the event subsystem (signals the completion/failure of bag/job-chains).
  63  */
  64
  65 static void __user *
  66 get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p)
  67 {
  68 #ifdef CONFIG_COMPAT
  69         if (kctx->is_compat)
  70                 return compat_ptr(p->compat_value);
  71 #endif
  72         return p->value;
  73 }
  74
  75 /* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
  76  *
  77  * Returns whether the JS needs a reschedule.
  78  *
  79  * Note that the caller must also check the atom status and
  80  * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock
  81  */
  82 static int jd_run_atom(struct kbase_jd_atom *katom)
  83 {
  84         struct kbase_context *kctx = katom->kctx;
  85
  86         KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);
  87
  88         if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) {
  89                 /* Dependency only atom */
  90                 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
  91                 return 0;
  92         } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
  93                 /* Soft-job */
  94                 if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
  95                                                   == BASE_JD_REQ_SOFT_REPLAY) {
  96                         if (!kbase_replay_process(katom))
  97                                 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
  98                 } else if (kbase_process_soft_job(katom) == 0) {
  99                         kbase_finish_soft_job(katom);
 100                         katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
 101                 } else {
 102                         /* The job has not completed */
 103                         list_add_tail(&katom->dep_item[0], &kctx->waiting_soft_jobs);
 104                 }
 105                 return 0;
 106         }
 107
 108         katom->status = KBASE_JD_ATOM_STATE_IN_JS;
 109         /* Queue an action about whether we should try scheduling a context */
 110         return kbasep_js_add_job(kctx, katom);
 111 }
 112
 113 #ifdef CONFIG_KDS
 114
 115 /* Add the katom to the kds waiting list.
 116  * Atoms must be added to the waiting list after a successful call to kds_async_waitall.
 117  * The caller must hold the kbase_jd_context.lock */
 118
 119 static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom)
 120 {
 121         struct kbase_context *kctx;
 122
 123         KBASE_DEBUG_ASSERT(katom);
 124
 125         kctx = katom->kctx;
 126
 127         list_add_tail(&katom->node, &kctx->waiting_kds_resource);
 128 }
 129
 130 /* Remove the katom from the kds waiting list.
 131  * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync.
 132  * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add.
 133  * The caller must hold the kbase_jd_context.lock */
 134
 135 static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom)
 136 {
 137         KBASE_DEBUG_ASSERT(katom);
 138         list_del(&katom->node);
 139 }
 140
 141 static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter)
 142 {
 143         struct kbase_jd_atom *katom;
 144         struct kbase_jd_context *ctx;
 145         struct kbase_device *kbdev;
 146
 147         katom = (struct kbase_jd_atom *)callback_parameter;
 148         KBASE_DEBUG_ASSERT(katom);
 149         ctx = &katom->kctx->jctx;
 150         kbdev = katom->kctx->kbdev;
 151         KBASE_DEBUG_ASSERT(kbdev);
 152
 153         mutex_lock(&ctx->lock);
 154
 155         /* KDS resource has already been satisfied (e.g. due to zapping) */
 156         if (katom->kds_dep_satisfied)
 157                 goto out;
 158
 159         /* This atom's KDS dependency has now been met */
 160         katom->kds_dep_satisfied = true;
 161
 162         /* Check whether the atom's other dependencies were already met. If
 163          * katom is a GPU atom then the job scheduler may be able to represent
 164          * the dependencies, hence we may attempt to submit it before they are
 165          * met. Other atoms must have had both dependencies resolved */
 166         if (IS_GPU_ATOM(katom) ||
 167                         (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
 168                         !kbase_jd_katom_dep_atom(&katom->dep[1]))) {
 169                 /* katom dep complete, attempt to run it */
 170                 bool resched = false;
 171
 172                 resched = jd_run_atom(katom);
 173
 174                 if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
 175                         /* The atom has already finished */
 176                         resched |= jd_done_nolock(katom, NULL);
 177                 }
 178
 179                 if (resched)
 180                         kbase_js_sched_all(kbdev);
 181         }
 182  out:
 183         mutex_unlock(&ctx->lock);
 184 }
 185
 186 static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom)
 187 {
 188         KBASE_DEBUG_ASSERT(katom);
 189
 190         /* Prevent job_done_nolock from being called twice on an atom when
 191          *  there is a race between job completion and cancellation */
 192
 193         if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) {
 194                 /* Wait was cancelled - zap the atom */
 195                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 196                 if (jd_done_nolock(katom, NULL))
 197                         kbase_js_sched_all(katom->kctx->kbdev);
 198         }
 199 }
 200 #endif                          /* CONFIG_KDS */
 201
 202 static int kbase_jd_user_buf_map(struct kbase_context *kctx,
 203                 struct kbase_va_region *reg)
 204 {
 205         long pinned_pages;
 206         struct kbase_mem_phy_alloc *alloc;
 207         struct page **pages;
 208         phys_addr_t *pa;
 209         long i;
 210         int err = -ENOMEM;
 211         unsigned long address;
 212         struct task_struct *owner;
 213         struct device *dev;
 214         unsigned long offset;
 215         unsigned long local_size;
 216
 217         alloc = reg->gpu_alloc;
 218         pa = kbase_get_gpu_phy_pages(reg);
 219         address = alloc->imported.user_buf.address;
 220         owner = alloc->imported.user_buf.owner;
 221
 222         KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
 223
 224         pages = alloc->imported.user_buf.pages;
 225
 226         down_read(&owner->mm->mmap_sem);
 227         pinned_pages = get_user_pages(owner, owner->mm,
 228                         address,
 229                         alloc->imported.user_buf.nr_pages,
 230                         reg->flags & KBASE_REG_GPU_WR,
 231                         0, pages, NULL);
 232         up_read(&owner->mm->mmap_sem);
 233
 234         if (pinned_pages <= 0)
 235                 return pinned_pages;
 236
 237         if (pinned_pages != alloc->imported.user_buf.nr_pages) {
 238                 for (i = 0; i < pinned_pages; i++)
 239                         put_page(pages[i]);
 240                 return -ENOMEM;
 241         }
 242
 243         dev = kctx->kbdev->dev;
 244         offset = address & ~PAGE_MASK;
 245         local_size = alloc->imported.user_buf.size;
 246
 247         for (i = 0; i < pinned_pages; i++) {
 248                 dma_addr_t dma_addr;
 249                 unsigned long min;
 250
 251                 min = MIN(PAGE_SIZE - offset, local_size);
 252                 dma_addr = dma_map_page(dev, pages[i],
 253                                 offset, min,
 254                                 DMA_BIDIRECTIONAL);
 255                 if (dma_mapping_error(dev, dma_addr))
 256                         goto unwind;
 257
 258                 alloc->imported.user_buf.dma_addrs[i] = dma_addr;
 259                 pa[i] = page_to_phys(pages[i]);
 260
 261                 local_size -= min;
 262                 offset = 0;
 263         }
 264
 265         alloc->nents = pinned_pages;
 266
 267         err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa,
 268                         kbase_reg_current_backed_size(reg),
 269                         reg->flags);
 270         if (err == 0)
 271                 return 0;
 272
 273         alloc->nents = 0;
 274         /* fall down */
 275 unwind:
 276         while (i--) {
 277                 dma_unmap_page(kctx->kbdev->dev,
 278                                 alloc->imported.user_buf.dma_addrs[i],
 279                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
 280                 put_page(pages[i]);
 281                 pages[i] = NULL;
 282         }
 283
 284         return err;
 285 }
 286
 287 static void kbase_jd_user_buf_unmap(struct kbase_context *kctx,
 288                 struct kbase_mem_phy_alloc *alloc, bool writeable)
 289 {
 290         long i;
 291         struct page **pages;
 292         unsigned long size = alloc->imported.user_buf.size;
 293
 294         KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
 295         pages = alloc->imported.user_buf.pages;
 296         for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) {
 297                 unsigned long local_size;
 298                 dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i];
 299
 300                 local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK));
 301                 dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size,
 302                                 DMA_BIDIRECTIONAL);
 303                 if (writeable)
 304                         set_page_dirty_lock(pages[i]);
 305                 put_page(pages[i]);
 306                 pages[i] = NULL;
 307
 308                 size -= local_size;
 309         }
 310         alloc->nents = 0;
 311 }
 312
 313 /* not to use sg_dma_len. */
 314 #define MALI_SG_DMA_LEN(sg)             ((sg)->length)
 315
 316 #ifdef CONFIG_DMA_SHARED_BUFFER
 317 static int kbase_jd_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg)
 318 {
 319         struct sg_table *sgt;   /* scatterlist_table */
 320         struct scatterlist *s;
 321         int i;
 322         phys_addr_t *pa;
 323         int err;
 324         size_t count = 0;
 325         struct kbase_mem_phy_alloc *alloc;
 326
 327         alloc = reg->gpu_alloc;
 328
 329         KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM);
 330         KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt);
 331         sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, DMA_BIDIRECTIONAL);
 332
 333         if (IS_ERR_OR_NULL(sgt))
 334                 return -EINVAL;
 335
 336         /* save for later */
 337         alloc->imported.umm.sgt = sgt;
 338
 339         pa = kbase_get_gpu_phy_pages(reg);
 340         KBASE_DEBUG_ASSERT(pa);
 341
 342         for_each_sg(sgt->sgl, s, sgt->nents, i) {
 343                 int j;
 344                 /* size_t pages = PFN_UP(sg_dma_len(s)); */
 345                 size_t pages = PFN_UP(MALI_SG_DMA_LEN(s));
 346
 347                 WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1),
 348                 "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n",
 349                 MALI_SG_DMA_LEN(s));
 350                 /*
 351                 WARN_ONCE(sg_dma_len(s) & (PAGE_SIZE-1),
 352                 "sg_dma_len(s)=%u is not a multiple of PAGE_SIZE\n",
 353                 sg_dma_len(s));
 354                 */
 355
 356                 WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1),
 357                 "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n",
 358                 (unsigned long long) sg_dma_address(s));
 359
 360                 for (j = 0; (j < pages) && (count < reg->nr_pages); j++, count++)
 361                         *pa++ = sg_dma_address(s) + (j << PAGE_SHIFT);
 362
 363                 WARN_ONCE(j < pages,
 364                 "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n",
 365                 alloc->imported.umm.dma_buf->size);
 366         }
 367
 368         if (WARN_ONCE(count < reg->nr_pages,
 369                         "sg list from dma_buf_map_attachment < dma_buf->size=%zu, count : %lu, reg->nr_pages : %lu. \n",
 370                         alloc->imported.umm.dma_buf->size,
 371                         count,
 372                         reg->nr_pages)) {
 373                 err = -EINVAL;
 374                 goto out;
 375         }
 376
 377         /* Update nents as we now have pages to map */
 378         alloc->nents = count;
 379
 380         err = kbase_mmu_insert_pages(kctx, reg->start_pfn, kbase_get_gpu_phy_pages(reg), kbase_reg_current_backed_size(reg), reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);
 381
 382 out:
 383         if (err) {
 384                 dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
 385                 alloc->imported.umm.sgt = NULL;
 386         }
 387
 388         return err;
 389 }
 390
 391 static void kbase_jd_umm_unmap(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc)
 392 {
 393         KBASE_DEBUG_ASSERT(kctx);
 394         KBASE_DEBUG_ASSERT(alloc);
 395         KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment);
 396         KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt);
 397         dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment,
 398             alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
 399         alloc->imported.umm.sgt = NULL;
 400         alloc->nents = 0;
 401 }
 402 #endif                          /* CONFIG_DMA_SHARED_BUFFER */
 403
 404 void kbase_jd_free_external_resources(struct kbase_jd_atom *katom)
 405 {
 406 #ifdef CONFIG_KDS
 407         if (katom->kds_rset) {
 408                 struct kbase_jd_context *jctx = &katom->kctx->jctx;
 409
 410                 /*
 411                  * As the atom is no longer waiting, remove it from
 412                  * the waiting list.
 413                  */
 414
 415                 mutex_lock(&jctx->lock);
 416                 kbase_jd_kds_waiters_remove(katom);
 417                 mutex_unlock(&jctx->lock);
 418
 419                 /* Release the kds resource or cancel if zapping */
 420                 kds_resource_set_release_sync(&katom->kds_rset);
 421         }
 422 #endif                          /* CONFIG_KDS */
 423 }
 424
 425 static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom)
 426 {
 427         KBASE_DEBUG_ASSERT(katom);
 428         KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);
 429
 430 #ifdef CONFIG_KDS
 431         /* Prevent the KDS resource from triggering the atom in case of zapping */
 432         if (katom->kds_rset)
 433                 katom->kds_dep_satisfied = true;
 434 #endif                          /* CONFIG_KDS */
 435
 436         kbase_gpu_vm_lock(katom->kctx);
 437         /* only roll back if extres is non-NULL */
 438         if (katom->extres) {
 439                 u32 res_no;
 440
 441                 res_no = katom->nr_extres;
 442                 while (res_no-- > 0) {
 443                         struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;
 444
 445                         switch (alloc->type) {
 446 #ifdef CONFIG_DMA_SHARED_BUFFER
 447                         case KBASE_MEM_TYPE_IMPORTED_UMM: {
 448                                 alloc->imported.umm.current_mapping_usage_count--;
 449
 450                                 if (0 == alloc->imported.umm.current_mapping_usage_count) {
 451                                         struct kbase_va_region *reg;
 452
 453                                         reg = kbase_region_tracker_find_region_base_address(
 454                                                         katom->kctx,
 455                                                         katom->extres[res_no].gpu_address);
 456
 457                                         if (reg && reg->gpu_alloc == alloc)
 458                                                 kbase_mmu_teardown_pages(
 459                                                                 katom->kctx,
 460                                                                 reg->start_pfn,
 461                                                                 kbase_reg_current_backed_size(reg));
 462
 463                                         kbase_jd_umm_unmap(katom->kctx, alloc);
 464                                 }
 465                         }
 466                         break;
 467 #endif /* CONFIG_DMA_SHARED_BUFFER */
 468                         case KBASE_MEM_TYPE_IMPORTED_USER_BUF: {
 469                                 alloc->imported.user_buf.current_mapping_usage_count--;
 470
 471                                 if (0 == alloc->imported.user_buf.current_mapping_usage_count) {
 472                                         struct kbase_va_region *reg;
 473
 474                                         reg = kbase_region_tracker_find_region_base_address(
 475                                                         katom->kctx,
 476                                                         katom->extres[res_no].gpu_address);
 477
 478                                         if (reg && reg->gpu_alloc == alloc)
 479                                                 kbase_mmu_teardown_pages(
 480                                                                 katom->kctx,
 481                                                                 reg->start_pfn,
 482                                                                 kbase_reg_current_backed_size(reg));
 483
 484                                         kbase_jd_user_buf_unmap(katom->kctx,
 485                                                         alloc,
 486                                                         reg->flags & KBASE_REG_GPU_WR);
 487                                 }
 488                         }
 489                         break;
 490                         default:
 491                         break;
 492                         }
 493                         kbase_mem_phy_alloc_put(katom->extres[res_no].alloc);
 494                 }
 495                 kfree(katom->extres);
 496                 katom->extres = NULL;
 497         }
 498         kbase_gpu_vm_unlock(katom->kctx);
 499 }
 500
 501 #if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS)
 502 static void add_kds_resource(struct kds_resource *kds_res, struct kds_resource **kds_resources, u32 *kds_res_count, unsigned long *kds_access_bitmap, bool exclusive)
 503 {
 504         u32 i;
 505
 506         for (i = 0; i < *kds_res_count; i++) {
 507                 /* Duplicate resource, ignore */
 508                 if (kds_resources[i] == kds_res)
 509                         return;
 510         }
 511
 512         kds_resources[*kds_res_count] = kds_res;
 513         if (exclusive)
 514                 set_bit(*kds_res_count, kds_access_bitmap);
 515         (*kds_res_count)++;
 516 }
 517 #endif
 518
 519 /*
 520  * Set up external resources needed by this job.
 521  *
 522  * jctx.lock must be held when this is called.
 523  */
 524
 525 static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom)
 526 {
 527         int err_ret_val = -EINVAL;
 528         u32 res_no;
 529 #ifdef CONFIG_KDS
 530         u32 kds_res_count = 0;
 531         struct kds_resource **kds_resources = NULL;
 532         unsigned long *kds_access_bitmap = NULL;
 533 #endif                          /* CONFIG_KDS */
 534         struct base_external_resource *input_extres;
 535
 536         KBASE_DEBUG_ASSERT(katom);
 537         KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);
 538
 539         /* no resources encoded, early out */
 540         if (!katom->nr_extres)
 541                 return -EINVAL;
 542
 543         katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL);
 544         if (NULL == katom->extres) {
 545                 err_ret_val = -ENOMEM;
 546                 goto early_err_out;
 547         }
 548
 549         /* copy user buffer to the end of our real buffer.
 550          * Make sure the struct sizes haven't changed in a way
 551          * we don't support */
 552         BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres));
 553         input_extres = (struct base_external_resource *)
 554                         (((unsigned char *)katom->extres) +
 555                         (sizeof(*katom->extres) - sizeof(*input_extres)) *
 556                         katom->nr_extres);
 557
 558         if (copy_from_user(input_extres,
 559                         get_compat_pointer(katom->kctx, &user_atom->extres_list),
 560                         sizeof(*input_extres) * katom->nr_extres) != 0) {
 561                 err_ret_val = -EINVAL;
 562                 goto early_err_out;
 563         }
 564 #ifdef CONFIG_KDS
 565         /* assume we have to wait for all */
 566         KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
 567         kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL);
 568
 569         if (NULL == kds_resources) {
 570                 err_ret_val = -ENOMEM;
 571                 goto early_err_out;
 572         }
 573
 574         KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
 575         kds_access_bitmap = kzalloc(sizeof(unsigned long) * ((katom->nr_extres + BITS_PER_LONG - 1) / BITS_PER_LONG), GFP_KERNEL);
 576
 577         if (NULL == kds_access_bitmap) {
 578                 err_ret_val = -ENOMEM;
 579                 goto early_err_out;
 580         }
 581 #endif                          /* CONFIG_KDS */
 582
 583         /* need to keep the GPU VM locked while we set up UMM buffers */
 584         kbase_gpu_vm_lock(katom->kctx);
 585         for (res_no = 0; res_no < katom->nr_extres; res_no++) {
 586                 struct base_external_resource *res;
 587                 struct kbase_va_region *reg;
 588
 589                 res = &input_extres[res_no];
 590                 reg = kbase_region_tracker_find_region_enclosing_address(
 591                                 katom->kctx,
 592                                 res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
 593                 /* did we find a matching region object? */
 594                 if (NULL == reg || (reg->flags & KBASE_REG_FREE)) {
 595                         /* roll back */
 596                         goto failed_loop;
 597                 }
 598
 599                 if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) &&
 600                                 (reg->flags & KBASE_REG_SECURE)) {
 601                         katom->atom_flags |= KBASE_KATOM_FLAG_SECURE;
 602                         if ((katom->core_req & BASE_JD_REQ_FS) == 0) {
 603                                 WARN_RATELIMIT(1, "Secure non-fragment jobs not supported");
 604                                 goto failed_loop;
 605                         }
 606                 }
 607
 608                 /* decide what needs to happen for this resource */
 609                 switch (reg->gpu_alloc->type) {
 610                 case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
 611                         reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++;
 612                         if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) {
 613                                         /* use a local variable to not pollute
 614                                          * err_ret_val with a potential success
 615                                          * value as some other gotos depend on
 616                                          * the default error code stored in
 617                                          * err_ret_val */
 618                                         int tmp;
 619
 620                                         tmp = kbase_jd_user_buf_map(katom->kctx,
 621                                                         reg);
 622                                         if (0 != tmp) {
 623                                                 /* failed to map this buffer,
 624                                                  * roll back */
 625                                                 err_ret_val = tmp;
 626                                                 reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--;
 627                                                 goto failed_loop;
 628                                         }
 629                         }
 630                 }
 631                 break;
 632                 case BASE_MEM_IMPORT_TYPE_UMP: {
 633 #if defined(CONFIG_KDS) && defined(CONFIG_UMP)
 634                                 struct kds_resource *kds_res;
 635
 636                                 kds_res = ump_dd_kds_resource_get(reg->gpu_alloc->imported.ump_handle);
 637                                 if (kds_res)
 638                                         add_kds_resource(kds_res, kds_resources, &kds_res_count,
 639                                                         kds_access_bitmap,
 640                                                         res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE);
 641 #endif                          /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */
 642                                 break;
 643                 }
 644 #ifdef CONFIG_DMA_SHARED_BUFFER
 645                 case BASE_MEM_IMPORT_TYPE_UMM: {
 646 #ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS
 647                                 struct kds_resource *kds_res;
 648
 649                                 kds_res = get_dma_buf_kds_resource(reg->gpu_alloc->imported.umm.dma_buf);
 650                                 if (kds_res)
 651                                         add_kds_resource(kds_res, kds_resources, &kds_res_count, kds_access_bitmap, res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE);
 652 #endif
 653                                 reg->gpu_alloc->imported.umm.current_mapping_usage_count++;
 654                                 if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) {
 655                                         /* use a local variable to not pollute err_ret_val
 656                                          * with a potential success value as some other gotos depend
 657                                          * on the default error code stored in err_ret_val */
 658                                         int tmp;
 659
 660                                         tmp = kbase_jd_umm_map(katom->kctx, reg);
 661                                         if (tmp) {
 662                                                 /* failed to map this buffer, roll back */
 663                                                 err_ret_val = tmp;
 664                                                 reg->gpu_alloc->imported.umm.current_mapping_usage_count--;
 665                                                 goto failed_loop;
 666                                         }
 667                                 }
 668                                 break;
 669                 }
 670 #endif
 671                 default:
 672                         goto failed_loop;
 673                 }
 674
 675                 /* finish with updating out array with the data we found */
 676                 /* NOTE: It is important that this is the last thing we do (or
 677                  * at least not before the first write) as we overwrite elements
 678                  * as we loop and could be overwriting ourself, so no writes
 679                  * until the last read for an element.
 680                  * */
 681                 katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */
 682                 katom->extres[res_no].alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
 683         }
 684         /* successfully parsed the extres array */
 685         /* drop the vm lock before we call into kds */
 686         kbase_gpu_vm_unlock(katom->kctx);
 687
 688 #ifdef CONFIG_KDS
 689         if (kds_res_count) {
 690                 int wait_failed;
 691
 692                 /* We have resources to wait for with kds */
 693                 katom->kds_dep_satisfied = false;
 694
 695                 wait_failed = kds_async_waitall(&katom->kds_rset,
 696                                 &katom->kctx->jctx.kds_cb, katom, NULL,
 697                                 kds_res_count, kds_access_bitmap,
 698                                 kds_resources);
 699
 700                 if (wait_failed)
 701                         goto failed_kds_setup;
 702                 else
 703                         kbase_jd_kds_waiters_add(katom);
 704         } else {
 705                 /* Nothing to wait for, so kds dep met */
 706                 katom->kds_dep_satisfied = true;
 707         }
 708         kfree(kds_resources);
 709         kfree(kds_access_bitmap);
 710 #endif                          /* CONFIG_KDS */
 711
 712         /* all done OK */
 713         return 0;
 714
 715 /* error handling section */
 716
 717 #ifdef CONFIG_KDS
 718  failed_kds_setup:
 719
 720         /* lock before we unmap */
 721         kbase_gpu_vm_lock(katom->kctx);
 722 #endif                          /* CONFIG_KDS */
 723
 724  failed_loop:
 725         /* undo the loop work */
 726         while (res_no-- > 0) {
 727                 struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;
 728 #ifdef CONFIG_DMA_SHARED_BUFFER
 729                 if (alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) {
 730                         alloc->imported.umm.current_mapping_usage_count--;
 731
 732                         if (0 == alloc->imported.umm.current_mapping_usage_count) {
 733                                 struct kbase_va_region *reg;
 734
 735                                 reg = kbase_region_tracker_find_region_base_address(
 736                                                 katom->kctx,
 737                                                 katom->extres[res_no].gpu_address);
 738
 739                                 if (reg && reg->gpu_alloc == alloc)
 740                                         kbase_mmu_teardown_pages(katom->kctx,
 741                                                         reg->start_pfn,
 742                                                         kbase_reg_current_backed_size(reg));
 743
 744                                 kbase_jd_umm_unmap(katom->kctx, alloc);
 745                         }
 746                 }
 747 #endif                          /* CONFIG_DMA_SHARED_BUFFER */
 748                 kbase_mem_phy_alloc_put(alloc);
 749         }
 750         kbase_gpu_vm_unlock(katom->kctx);
 751
 752  early_err_out:
 753         kfree(katom->extres);
 754         katom->extres = NULL;
 755 #ifdef CONFIG_KDS
 756         kfree(kds_resources);
 757         kfree(kds_access_bitmap);
 758 #endif                          /* CONFIG_KDS */
 759         return err_ret_val;
 760 }
 761
 762 static inline void jd_resolve_dep(struct list_head *out_list,
 763                                         struct kbase_jd_atom *katom,
 764                                         u8 d,
 765                                         bool ctx_is_dying)
 766 {
 767         u8 other_d = !d;
 768
 769         while (!list_empty(&katom->dep_head[d])) {
 770                 struct kbase_jd_atom *dep_atom;
 771                 u8 dep_type;
 772
 773                 dep_atom = list_entry(katom->dep_head[d].next,
 774                                 struct kbase_jd_atom, dep_item[d]);
 775
 776                 list_del(katom->dep_head[d].next);
 777
 778                 dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]);
 779                 kbase_jd_katom_dep_clear(&dep_atom->dep[d]);
 780
 781                 if (katom->event_code != BASE_JD_EVENT_DONE &&
 782                         (dep_type != BASE_JD_DEP_TYPE_ORDER || ctx_is_dying)) {
 783                         /* Atom failed, so remove the other dependencies and immediately fail the atom */
 784                         if (kbase_jd_katom_dep_atom(&dep_atom->dep[other_d])) {
 785                                 list_del(&dep_atom->dep_item[other_d]);
 786                                 kbase_jd_katom_dep_clear(&dep_atom->dep[other_d]);
 787                         }
 788 #ifdef CONFIG_KDS
 789                         if (!dep_atom->kds_dep_satisfied) {
 790                                 /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and
 791                                  * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up
 792                                  */
 793                                 dep_atom->kds_dep_satisfied = true;
 794                         }
 795 #endif
 796
 797                         dep_atom->event_code = katom->event_code;
 798                         KBASE_DEBUG_ASSERT(dep_atom->status !=
 799                                                 KBASE_JD_ATOM_STATE_UNUSED);
 800                         dep_atom->status = KBASE_JD_ATOM_STATE_COMPLETED;
 801
 802                         list_add_tail(&dep_atom->dep_item[0], out_list);
 803                 } else if (!kbase_jd_katom_dep_atom(&dep_atom->dep[other_d])) {
 804 #ifdef CONFIG_KDS
 805                         if (dep_atom->kds_dep_satisfied)
 806 #endif
 807                                 list_add_tail(&dep_atom->dep_item[0], out_list);
 808                 }
 809         }
 810 }
 811
 812 KBASE_EXPORT_TEST_API(jd_resolve_dep);
 813
 814 #if MALI_CUSTOMER_RELEASE == 0
 815 static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
 816 {
 817         kbdev->force_replay_count++;
 818
 819         if (kbdev->force_replay_count >= kbdev->force_replay_limit) {
 820                 kbdev->force_replay_count = 0;
 821                 katom->event_code = BASE_JD_EVENT_FORCE_REPLAY;
 822
 823                 if (kbdev->force_replay_random)
 824                         kbdev->force_replay_limit =
 825                            (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1;
 826
 827                 dev_info(kbdev->dev, "force_replay : promoting to error\n");
 828         }
 829 }
 830
 831 /** Test to see if atom should be forced to fail.
 832  *
 833  * This function will check if an atom has a replay job as a dependent. If so
 834  * then it will be considered for forced failure. */
 835 static void jd_check_force_failure(struct kbase_jd_atom *katom)
 836 {
 837         struct kbase_context *kctx = katom->kctx;
 838         struct kbase_device *kbdev = kctx->kbdev;
 839         int i;
 840
 841         if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) ||
 842             (katom->core_req & BASEP_JD_REQ_EVENT_NEVER))
 843                 return;
 844
 845         for (i = 1; i < BASE_JD_ATOM_COUNT; i++) {
 846                 if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom ||
 847                     kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) {
 848                         struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i];
 849
 850                         if ((dep_atom->core_req & BASEP_JD_REQ_ATOM_TYPE) ==
 851                                                      BASE_JD_REQ_SOFT_REPLAY &&
 852                             (dep_atom->core_req & kbdev->force_replay_core_req)
 853                                              == kbdev->force_replay_core_req) {
 854                                 jd_force_failure(kbdev, katom);
 855                                 return;
 856                         }
 857                 }
 858         }
 859 }
 860 #endif
 861
 862 /*
 863  * Perform the necessary handling of an atom that has finished running
 864  * on the GPU.
 865  *
 866  * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller
 867  * is responsible for calling kbase_finish_soft_job *before* calling this function.
 868  *
 869  * The caller must hold the kbase_jd_context.lock.
 870  */
 871 bool jd_done_nolock(struct kbase_jd_atom *katom,
 872                 struct list_head *completed_jobs_ctx)
 873 {
 874         struct kbase_context *kctx = katom->kctx;
 875         struct kbase_device *kbdev = kctx->kbdev;
 876         struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
 877         struct list_head completed_jobs;
 878         struct list_head runnable_jobs;
 879         bool need_to_try_schedule_context = false;
 880         int i;
 881
 882         INIT_LIST_HEAD(&completed_jobs);
 883         INIT_LIST_HEAD(&runnable_jobs);
 884
 885         KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);
 886
 887 #if MALI_CUSTOMER_RELEASE == 0
 888         jd_check_force_failure(katom);
 889 #endif
 890
 891
 892         /* This is needed in case an atom is failed due to being invalid, this
 893          * can happen *before* the jobs that the atom depends on have completed */
 894         for (i = 0; i < 2; i++) {
 895                 if (kbase_jd_katom_dep_atom(&katom->dep[i])) {
 896                         list_del(&katom->dep_item[i]);
 897                         kbase_jd_katom_dep_clear(&katom->dep[i]);
 898                 }
 899         }
 900
 901         /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with
 902          * BASE_JD_EVENT_TILE_RANGE_FAULT.
 903          *
 904          * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the
 905          * error code to BASE_JD_EVENT_DONE
 906          */
 907
 908         if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) &&
 909                   katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) {
 910                 if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) {
 911                         /* Promote the failure to job done */
 912                         katom->event_code = BASE_JD_EVENT_DONE;
 913                         katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED);
 914                 }
 915         }
 916
 917         katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
 918         list_add_tail(&katom->dep_item[0], &completed_jobs);
 919
 920         while (!list_empty(&completed_jobs)) {
 921                 katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, dep_item[0]);
 922                 list_del(completed_jobs.prev);
 923
 924                 KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED);
 925
 926                 for (i = 0; i < 2; i++)
 927                         jd_resolve_dep(&runnable_jobs, katom, i,
 928                                                 js_kctx_info->ctx.is_dying);
 929
 930                 if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
 931                         kbase_jd_post_external_resources(katom);
 932
 933                 while (!list_empty(&runnable_jobs)) {
 934                         struct kbase_jd_atom *node;
 935
 936                         node = list_entry(runnable_jobs.next,
 937                                         struct kbase_jd_atom, dep_item[0]);
 938
 939                         list_del(runnable_jobs.next);
 940
 941                         KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);
 942
 943                         if (node->status != KBASE_JD_ATOM_STATE_COMPLETED) {
 944                                 need_to_try_schedule_context |= jd_run_atom(node);
 945                         } else {
 946                                 node->event_code = katom->event_code;
 947
 948                                 if ((node->core_req & BASEP_JD_REQ_ATOM_TYPE)
 949                                                   == BASE_JD_REQ_SOFT_REPLAY) {
 950                                         if (kbase_replay_process(node))
 951                                                 /* Don't complete this atom */
 952                                                 continue;
 953                                 } else if (node->core_req &
 954                                                         BASE_JD_REQ_SOFT_JOB) {
 955                                         /* If this is a fence wait then remove it from the list of sync waiters. */
 956                                         if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req)
 957                                                 list_del(&node->dep_item[0]);
 958
 959                                         kbase_finish_soft_job(node);
 960                                 }
 961                                 node->status = KBASE_JD_ATOM_STATE_COMPLETED;
 962                         }
 963
 964                         if (node->status == KBASE_JD_ATOM_STATE_COMPLETED)
 965                                 list_add_tail(&node->dep_item[0], &completed_jobs);
 966                 }
 967
 968                 /* Register a completed job as a disjoint event when the GPU
 969                  * is in a disjoint state (ie. being reset or replaying jobs).
 970                  */
 971                 kbase_disjoint_event_potential(kctx->kbdev);
 972                 if (completed_jobs_ctx)
 973                         list_add_tail(&katom->dep_item[0], completed_jobs_ctx);
 974                 else
 975                         kbase_event_post(kctx, katom);
 976
 977                 /* Decrement and check the TOTAL number of jobs. This includes
 978                  * those not tracked by the scheduler: 'not ready to run' and
 979                  * 'dependency-only' jobs. */
 980                 if (--kctx->jctx.job_nr == 0)
 981                         wake_up(&kctx->jctx.zero_jobs_wait);    /* All events are safely queued now, and we can signal any waiter
 982                                                                  * that we've got no more jobs (so we can be safely terminated) */
 983         }
 984
 985         return need_to_try_schedule_context;
 986 }
 987
 988 KBASE_EXPORT_TEST_API(jd_done_nolock);
 989
 990 #ifdef CONFIG_GPU_TRACEPOINTS
 991 enum {
 992         CORE_REQ_DEP_ONLY,
 993         CORE_REQ_SOFT,
 994         CORE_REQ_COMPUTE,
 995         CORE_REQ_FRAGMENT,
 996         CORE_REQ_VERTEX,
 997         CORE_REQ_TILER,
 998         CORE_REQ_FRAGMENT_VERTEX,
 999         CORE_REQ_FRAGMENT_VERTEX_TILER,
1000         CORE_REQ_FRAGMENT_TILER,
1001         CORE_REQ_VERTEX_TILER,
1002         CORE_REQ_UNKNOWN
1003 };
1004 static const char * const core_req_strings[] = {
1005         "Dependency Only Job",
1006         "Soft Job",
1007         "Compute Shader Job",
1008         "Fragment Shader Job",
1009         "Vertex/Geometry Shader Job",
1010         "Tiler Job",
1011         "Fragment Shader + Vertex/Geometry Shader Job",
1012         "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job",
1013         "Fragment Shader + Tiler Job",
1014         "Vertex/Geometry Shader Job + Tiler Job",
1015         "Unknown Job"
1016 };
1017 static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req)
1018 {
1019         if (core_req & BASE_JD_REQ_SOFT_JOB)
1020                 return core_req_strings[CORE_REQ_SOFT];
1021         if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
1022                 return core_req_strings[CORE_REQ_COMPUTE];
1023         switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) {
1024         case BASE_JD_REQ_DEP:
1025                 return core_req_strings[CORE_REQ_DEP_ONLY];
1026         case BASE_JD_REQ_FS:
1027                 return core_req_strings[CORE_REQ_FRAGMENT];
1028         case BASE_JD_REQ_CS:
1029                 return core_req_strings[CORE_REQ_VERTEX];
1030         case BASE_JD_REQ_T:
1031                 return core_req_strings[CORE_REQ_TILER];
1032         case (BASE_JD_REQ_FS | BASE_JD_REQ_CS):
1033                 return core_req_strings[CORE_REQ_FRAGMENT_VERTEX];
1034         case (BASE_JD_REQ_FS | BASE_JD_REQ_T):
1035                 return core_req_strings[CORE_REQ_FRAGMENT_TILER];
1036         case (BASE_JD_REQ_CS | BASE_JD_REQ_T):
1037                 return core_req_strings[CORE_REQ_VERTEX_TILER];
1038         case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T):
1039                 return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER];
1040         }
1041         return core_req_strings[CORE_REQ_UNKNOWN];
1042 }
1043 #endif
1044
1045 bool jd_submit_atom(struct kbase_context *kctx,
1046                          const struct base_jd_atom_v2 *user_atom,
1047                          struct kbase_jd_atom *katom)
1048 {
1049         struct kbase_jd_context *jctx = &kctx->jctx;
1050         base_jd_core_req core_req;
1051         int queued = 0;
1052         int i;
1053         int sched_prio;
1054         bool ret;
1055
1056         /* Update the TOTAL number of jobs. This includes those not tracked by
1057          * the scheduler: 'not ready to run' and 'dependency-only' jobs. */
1058         jctx->job_nr++;
1059
1060         core_req = user_atom->core_req;
1061
1062         katom->start_timestamp.tv64 = 0;
1063         katom->time_spent_us = 0;
1064         katom->udata = user_atom->udata;
1065         katom->kctx = kctx;
1066         katom->nr_extres = user_atom->nr_extres;
1067         katom->extres = NULL;
1068         katom->device_nr = user_atom->device_nr;
1069         katom->affinity = 0;
1070         katom->jc = user_atom->jc;
1071         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1072         katom->core_req = core_req;
1073         katom->atom_flags = 0;
1074         katom->retry_count = 0;
1075         katom->need_cache_flush_cores_retained = 0;
1076         katom->x_pre_dep = NULL;
1077         katom->x_post_dep = NULL;
1078 #ifdef CONFIG_KDS
1079         /* Start by assuming that the KDS dependencies are satisfied,
1080          * kbase_jd_pre_external_resources will correct this if there are dependencies */
1081         katom->kds_dep_satisfied = true;
1082         katom->kds_rset = NULL;
1083 #endif                          /* CONFIG_KDS */
1084
1085         /* Don't do anything if there is a mess up with dependencies.
1086            This is done in a separate cycle to check both the dependencies at ones, otherwise
1087            it will be extra complexity to deal with 1st dependency ( just added to the list )
1088            if only the 2nd one has invalid config.
1089          */
1090         for (i = 0; i < 2; i++) {
1091                 int dep_atom_number = user_atom->pre_dep[i].atom_id;
1092                 base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type;
1093
1094                 if (dep_atom_number) {
1095                         if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER &&
1096                                         dep_atom_type != BASE_JD_DEP_TYPE_DATA) {
1097                                 katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT;
1098                                 katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
1099 #if defined(CONFIG_MALI_MIPE_ENABLED)
1100                                 /* Wrong dependency setup. Atom will be sent
1101                                  * back to user space. Do not record any
1102                                  * dependencies. */
1103                                 kbase_tlstream_tl_new_atom(
1104                                                 katom,
1105                                                 kbase_jd_atom_id(kctx, katom));
1106                                 kbase_tlstream_tl_ret_atom_ctx(
1107                                                 katom, kctx);
1108 #endif
1109                                 ret = jd_done_nolock(katom, NULL);
1110                                 goto out;
1111                         }
1112                 }
1113         }
1114
1115         /* Add dependencies */
1116         for (i = 0; i < 2; i++) {
1117                 int dep_atom_number = user_atom->pre_dep[i].atom_id;
1118                 base_jd_dep_type dep_atom_type;
1119                 struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];
1120
1121                 dep_atom_type = user_atom->pre_dep[i].dependency_type;
1122                 kbase_jd_katom_dep_clear(&katom->dep[i]);
1123
1124                 if (!dep_atom_number)
1125                         continue;
1126
1127                 if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED ||
1128                                 dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
1129
1130                         if (dep_atom->event_code == BASE_JD_EVENT_DONE)
1131                                 continue;
1132                         /* don't stop this atom if it has an order dependency
1133                          * only to the failed one, try to submit it through
1134                          * the normal path
1135                          */
1136                         if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER &&
1137                                         dep_atom->event_code > BASE_JD_EVENT_ACTIVE) {
1138                                 continue;
1139                         }
1140
1141                         if (i == 1 && kbase_jd_katom_dep_atom(&katom->dep[0])) {
1142                                 /* Remove the previous dependency */
1143                                 list_del(&katom->dep_item[0]);
1144                                 kbase_jd_katom_dep_clear(&katom->dep[0]);
1145                         }
1146
1147                         /* Atom has completed, propagate the error code if any */
1148                         katom->event_code = dep_atom->event_code;
1149                         katom->status = KBASE_JD_ATOM_STATE_QUEUED;
1150 #if defined(CONFIG_MALI_MIPE_ENABLED)
1151                         /* This atom is going through soft replay or
1152                          * will be sent back to user space. Do not record any
1153                          * dependencies. */
1154                         kbase_tlstream_tl_new_atom(
1155                                         katom,
1156                                         kbase_jd_atom_id(kctx, katom));
1157                         kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
1158 #endif
1159                         if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
1160                                          == BASE_JD_REQ_SOFT_REPLAY) {
1161                                 if (kbase_replay_process(katom)) {
1162                                         ret = false;
1163                                         goto out;
1164                                 }
1165                         }
1166                         ret = jd_done_nolock(katom, NULL);
1167
1168                         goto out;
1169                 } else {
1170                         /* Atom is in progress, add this atom to the list */
1171                         list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
1172                         kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type);
1173                         queued = 1;
1174                 }
1175         }
1176
1177         /* These must occur after the above loop to ensure that an atom that
1178          * depends on a previous atom with the same number behaves as expected */
1179         katom->event_code = BASE_JD_EVENT_DONE;
1180         katom->status = KBASE_JD_ATOM_STATE_QUEUED;
1181
1182 #if defined(CONFIG_MALI_MIPE_ENABLED)
1183         /* Create a new atom recording all dependencies it was set up with. */
1184         kbase_tlstream_tl_new_atom(
1185                         katom,
1186                         kbase_jd_atom_id(kctx, katom));
1187         kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
1188         for (i = 0; i < 2; i++)
1189                 if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type(
1190                                         &katom->dep[i]))
1191                         kbase_tlstream_tl_dep_atom_atom(
1192                                         (void *)kbase_jd_katom_dep_atom(
1193                                                 &katom->dep[i]),
1194                                         (void *)katom);
1195 #endif
1196
1197         /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */
1198         if (!katom->jc && (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
1199                 dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL");
1200                 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1201                 ret = jd_done_nolock(katom, NULL);
1202                 goto out;
1203         }
1204
1205         /* Reject atoms with an invalid device_nr */
1206         if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) &&
1207             (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) {
1208                 dev_warn(kctx->kbdev->dev,
1209                                 "Rejecting atom with invalid device_nr %d",
1210                                 katom->device_nr);
1211                 katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1212                 ret = jd_done_nolock(katom, NULL);
1213                 goto out;
1214         }
1215
1216         /* For invalid priority, be most lenient and choose the default */
1217         sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
1218         if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
1219                 sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
1220         katom->sched_priority = sched_prio;
1221
1222         if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
1223                 /* handle what we need to do to access the external resources */
1224                 if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
1225                         /* setup failed (no access, bad resource, unknown resource types, etc.) */
1226                         katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1227                         ret = jd_done_nolock(katom, NULL);
1228                         goto out;
1229                 }
1230         }
1231
1232         /* Validate the atom. Function will return error if the atom is
1233          * malformed.
1234          *
1235          * Soft-jobs never enter the job scheduler but have their own initialize method.
1236          *
1237          * If either fail then we immediately complete the atom with an error.
1238          */
1239         if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
1240                 if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) {
1241                         katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1242                         ret = jd_done_nolock(katom, NULL);
1243                         goto out;
1244                 }
1245         } else {
1246                 /* Soft-job */
1247                 if (kbase_prepare_soft_job(katom) != 0) {
1248                         katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1249                         ret = jd_done_nolock(katom, NULL);
1250                         goto out;
1251                 }
1252         }
1253
1254 #ifdef CONFIG_GPU_TRACEPOINTS
1255         katom->work_id = atomic_inc_return(&jctx->work_id);
1256         trace_gpu_job_enqueue((u32)kctx->id, katom->work_id,
1257                         kbasep_map_core_reqs_to_string(katom->core_req));
1258 #endif
1259
1260         if (queued && !IS_GPU_ATOM(katom)) {
1261                 ret = false;
1262                 goto out;
1263         }
1264 #ifdef CONFIG_KDS
1265         if (!katom->kds_dep_satisfied) {
1266                 /* Queue atom due to KDS dependency */
1267                 ret = false;
1268                 goto out;
1269         }
1270 #endif                          /* CONFIG_KDS */
1271
1272         if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
1273                                                   == BASE_JD_REQ_SOFT_REPLAY) {
1274                 if (kbase_replay_process(katom))
1275                         ret = false;
1276                 else
1277                         ret = jd_done_nolock(katom, NULL);
1278
1279                 goto out;
1280         } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
1281                 if (kbase_process_soft_job(katom) == 0) {
1282                         kbase_finish_soft_job(katom);
1283                         ret = jd_done_nolock(katom, NULL);
1284                         goto out;
1285                 }
1286                 /* The job has not yet completed */
1287                 list_add_tail(&katom->dep_item[0], &kctx->waiting_soft_jobs);
1288                 ret = false;
1289         } else if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
1290                 katom->status = KBASE_JD_ATOM_STATE_IN_JS;
1291                 ret = kbasep_js_add_job(kctx, katom);
1292                 /* If job was cancelled then resolve immediately */
1293                 if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED)
1294                         ret = jd_done_nolock(katom, NULL);
1295         } else {
1296                 /* This is a pure dependency. Resolve it immediately */
1297                 ret = jd_done_nolock(katom, NULL);
1298         }
1299
1300  out:
1301         return ret;
1302 }
1303
1304 #ifdef BASE_LEGACY_UK6_SUPPORT
1305 int kbase_jd_submit(struct kbase_context *kctx,
1306                 const struct kbase_uk_job_submit *submit_data,
1307                 int uk6_atom)
1308 #else
1309 int kbase_jd_submit(struct kbase_context *kctx,
1310                 const struct kbase_uk_job_submit *submit_data)
1311 #endif /* BASE_LEGACY_UK6_SUPPORT */
1312 {
1313         struct kbase_jd_context *jctx = &kctx->jctx;
1314         int err = 0;
1315         int i;
1316         bool need_to_try_schedule_context = false;
1317         struct kbase_device *kbdev;
1318         void __user *user_addr;
1319         u32 latest_flush;
1320
1321         /*
1322          * kbase_jd_submit isn't expected to fail and so all errors with the jobs
1323          * are reported by immediately falling them (through event system)
1324          */
1325         kbdev = kctx->kbdev;
1326
1327         beenthere(kctx, "%s", "Enter");
1328
1329         if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != 0) {
1330                 dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it");
1331                 return -EINVAL;
1332         }
1333
1334 #ifdef BASE_LEGACY_UK6_SUPPORT
1335         if ((uk6_atom && submit_data->stride !=
1336                         sizeof(struct base_jd_atom_v2_uk6)) ||
1337                         submit_data->stride != sizeof(base_jd_atom_v2)) {
1338 #else
1339         if (submit_data->stride != sizeof(base_jd_atom_v2)) {
1340 #endif /* BASE_LEGACY_UK6_SUPPORT */
1341                 dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel");
1342                 return -EINVAL;
1343         }
1344
1345         user_addr = get_compat_pointer(kctx, &submit_data->addr);
1346
1347         KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(submit_data->nr_atoms, &kctx->timeline.jd_atoms_in_flight));
1348
1349         /* All atoms submitted in this call have the same flush ID */
1350         latest_flush = kbase_backend_get_current_flush_id(kbdev);
1351
1352         for (i = 0; i < submit_data->nr_atoms; i++) {
1353                 struct base_jd_atom_v2 user_atom;
1354                 struct kbase_jd_atom *katom;
1355
1356 #ifdef BASE_LEGACY_UK6_SUPPORT
1357                 if (uk6_atom) {
1358                         struct base_jd_atom_v2_uk6 user_atom_v6;
1359                         base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA};
1360
1361                         if (copy_from_user(&user_atom_v6, user_addr,
1362                                         sizeof(user_atom_v6))) {
1363                                 err = -EINVAL;
1364                                 KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
1365                                         atomic_sub_return(
1366                                         submit_data->nr_atoms - i,
1367                                         &kctx->timeline.jd_atoms_in_flight));
1368                                 break;
1369                         }
1370                         /* Convert from UK6 atom format to UK7 format */
1371                         user_atom.jc = user_atom_v6.jc;
1372                         user_atom.udata = user_atom_v6.udata;
1373                         user_atom.extres_list = user_atom_v6.extres_list;
1374                         user_atom.nr_extres = user_atom_v6.nr_extres;
1375                         user_atom.core_req = user_atom_v6.core_req;
1376
1377                         /* atom number 0 is used for no dependency atoms */
1378                         if (!user_atom_v6.pre_dep[0])
1379                                 dep_types[0] = BASE_JD_DEP_TYPE_INVALID;
1380
1381                         base_jd_atom_dep_set(&user_atom.pre_dep[0],
1382                                         user_atom_v6.pre_dep[0],
1383                                         dep_types[0]);
1384
1385                         /* atom number 0 is used for no dependency atoms */
1386                         if (!user_atom_v6.pre_dep[1])
1387                                 dep_types[1] = BASE_JD_DEP_TYPE_INVALID;
1388
1389                         base_jd_atom_dep_set(&user_atom.pre_dep[1],
1390                                         user_atom_v6.pre_dep[1],
1391                                         dep_types[1]);
1392
1393                         user_atom.atom_number = user_atom_v6.atom_number;
1394                         user_atom.prio = user_atom_v6.prio;
1395                         user_atom.device_nr = user_atom_v6.device_nr;
1396                 } else {
1397 #endif /* BASE_LEGACY_UK6_SUPPORT */
1398                 if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) {
1399                         err = -EINVAL;
1400                         KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(submit_data->nr_atoms - i, &kctx->timeline.jd_atoms_in_flight));
1401                         break;
1402                 }
1403 #ifdef BASE_LEGACY_UK6_SUPPORT
1404                 }
1405 #endif /* BASE_LEGACY_UK6_SUPPORT */
1406
1407                 user_addr = (void __user *)((uintptr_t) user_addr + submit_data->stride);
1408
1409                 mutex_lock(&jctx->lock);
1410 #ifndef compiletime_assert
1411 #define compiletime_assert_defined
1412 #define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \
1413 while (false)
1414 #endif
1415                 compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) ==
1416                                         BASE_JD_ATOM_COUNT,
1417                         "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
1418                 compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) ==
1419                                         sizeof(user_atom.atom_number),
1420                         "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
1421 #ifdef compiletime_assert_defined
1422 #undef compiletime_assert
1423 #undef compiletime_assert_defined
1424 #endif
1425                 katom = &jctx->atoms[user_atom.atom_number];
1426
1427                 /* Record the flush ID for the cache flush optimisation */
1428                 katom->flush_id = latest_flush;
1429
1430                 while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) {
1431                         /* Atom number is already in use, wait for the atom to
1432                          * complete
1433                          */
1434                         mutex_unlock(&jctx->lock);
1435
1436                         /* This thread will wait for the atom to complete. Due
1437                          * to thread scheduling we are not sure that the other
1438                          * thread that owns the atom will also schedule the
1439                          * context, so we force the scheduler to be active and
1440                          * hence eventually schedule this context at some point
1441                          * later.
1442                          */
1443                         kbase_js_sched_all(kbdev);
1444
1445                         if (wait_event_killable(katom->completed,
1446                                         katom->status ==
1447                                         KBASE_JD_ATOM_STATE_UNUSED) != 0) {
1448                                 /* We're being killed so the result code
1449                                  * doesn't really matter
1450                                  */
1451                                 return 0;
1452                         }
1453                         mutex_lock(&jctx->lock);
1454                 }
1455
1456                 need_to_try_schedule_context |=
1457                                        jd_submit_atom(kctx, &user_atom, katom);
1458
1459                 /* Register a completed job as a disjoint event when the GPU is in a disjoint state
1460                  * (ie. being reset or replaying jobs).
1461                  */
1462                 kbase_disjoint_event_potential(kbdev);
1463
1464                 mutex_unlock(&jctx->lock);
1465         }
1466
1467         if (need_to_try_schedule_context)
1468                 kbase_js_sched_all(kbdev);
1469
1470         return err;
1471 }
1472
1473 KBASE_EXPORT_TEST_API(kbase_jd_submit);
1474
1475 void kbase_jd_done_worker(struct work_struct *data)
1476 {
1477         struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
1478         struct kbase_jd_context *jctx;
1479         struct kbase_context *kctx;
1480         struct kbasep_js_kctx_info *js_kctx_info;
1481         union kbasep_js_policy *js_policy;
1482         struct kbase_device *kbdev;
1483         struct kbasep_js_device_data *js_devdata;
1484         u64 cache_jc = katom->jc;
1485         struct kbasep_js_atom_retained_state katom_retained_state;
1486         bool schedule = false;
1487         bool context_idle;
1488         base_jd_core_req core_req = katom->core_req;
1489         u64 affinity = katom->affinity;
1490         enum kbase_atom_coreref_state coreref_state = katom->coreref_state;
1491
1492         /* Soft jobs should never reach this function */
1493         KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
1494
1495         kctx = katom->kctx;
1496         jctx = &kctx->jctx;
1497         kbdev = kctx->kbdev;
1498         js_kctx_info = &kctx->jctx.sched_info;
1499         js_devdata = &kbdev->js_data;
1500         js_policy = &kbdev->js_data.policy;
1501
1502         KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
1503
1504         kbase_backend_complete_wq(kbdev, katom);
1505
1506         /*
1507          * Begin transaction on JD context and JS context
1508          */
1509         mutex_lock(&jctx->lock);
1510         mutex_lock(&js_devdata->queue_mutex);
1511         mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
1512
1513         /* This worker only gets called on contexts that are scheduled *in*. This is
1514          * because it only happens in response to an IRQ from a job that was
1515          * running.
1516          */
1517         KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
1518
1519         if (katom->event_code == BASE_JD_EVENT_STOPPED) {
1520                 /* Atom has been promoted to stopped */
1521                 unsigned long flags;
1522
1523                 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1524                 mutex_unlock(&js_devdata->queue_mutex);
1525                 mutex_unlock(&jctx->lock);
1526
1527                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1528
1529                 katom->status = KBASE_JD_ATOM_STATE_IN_JS;
1530                 kbase_js_unpull(kctx, katom);
1531
1532                 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1533
1534                 return;
1535         }
1536
1537         if (katom->event_code != BASE_JD_EVENT_DONE)
1538                 dev_err(kbdev->dev,
1539                         "t6xx: GPU fault 0x%02lx from job slot %d\n",
1540                                         (unsigned long)katom->event_code,
1541                                                                 katom->slot_nr);
1542
1543         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
1544                 kbase_as_poking_timer_release_atom(kbdev, kctx, katom);
1545
1546         /* Retain state before the katom disappears */
1547         kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
1548
1549         if (!kbasep_js_has_atom_finished(&katom_retained_state)) {
1550                 mutex_lock(&js_devdata->runpool_mutex);
1551                 kbasep_js_clear_job_retry_submit(katom);
1552                 /* An atom that has been hard-stopped might have previously
1553                  * been soft-stopped and has just finished before the hard-stop
1554                  * occurred. For this reason, clear the hard-stopped flag */
1555                 katom->atom_flags &= ~(KBASE_KATOM_FLAG_BEEN_HARD_STOPPED);
1556                 mutex_unlock(&js_devdata->runpool_mutex);
1557         }
1558
1559         if (kbasep_js_has_atom_finished(&katom_retained_state))
1560                 schedule = true;
1561
1562         context_idle = kbase_js_complete_atom_wq(kctx, katom);
1563
1564         KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state));
1565
1566         kbasep_js_remove_job(kbdev, kctx, katom);
1567         mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1568         mutex_unlock(&js_devdata->queue_mutex);
1569         katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
1570         /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
1571         schedule |= jd_done_nolock(katom, &kctx->completed_jobs);
1572
1573         /* katom may have been freed now, do not use! */
1574
1575         if (context_idle) {
1576                 unsigned long flags;
1577
1578                 mutex_lock(&js_devdata->queue_mutex);
1579                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1580
1581                 /* If kbase_sched() has scheduled this context back in then
1582                  * ctx_active will have been set after we marked it as inactive,
1583                  * and another pm reference will have been taken, so drop our
1584                  * reference. But do not call kbase_jm_idle_ctx(), as the
1585                  * context is active and fast-starting is allowed.
1586                  *
1587                  * If an atom has been fast-started then kctx->atoms_pulled will
1588                  * be non-zero but ctx_active will still be false (as the
1589                  * previous pm reference has been inherited). Do NOT drop our
1590                  * reference, as it has been re-used, and leave the context as
1591                  * active.
1592                  *
1593                  * If no new atoms have been started then ctx_active will still
1594                  * be false and atoms_pulled will be zero, so drop the reference
1595                  * and call kbase_jm_idle_ctx().
1596                  *
1597                  * As the checks are done under both the queue_mutex and
1598                  * runpool_irq.lock is should be impossible for this to race
1599                  * with the scheduler code.
1600                  */
1601                 if (kctx->ctx_active || !atomic_read(&kctx->atoms_pulled)) {
1602                         /* Calling kbase_jm_idle_ctx() here will ensure that
1603                          * atoms are not fast-started when we drop the
1604                          * runpool_irq.lock. This is not performed if ctx_active
1605                          * is set as in that case another pm reference has been
1606                          * taken and a fast-start would be valid.
1607                          */
1608                         if (!kctx->ctx_active)
1609                                 kbase_jm_idle_ctx(kbdev, kctx);
1610                         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
1611                                         flags);
1612
1613                         kbase_pm_context_idle(kbdev);
1614                 } else {
1615                         kctx->ctx_active = true;
1616                         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
1617                                         flags);
1618                 }
1619                 mutex_unlock(&js_devdata->queue_mutex);
1620         }
1621
1622         /*
1623          * Transaction complete
1624          */
1625         mutex_unlock(&jctx->lock);
1626
1627         /* Job is now no longer running, so can now safely release the context
1628          * reference, and handle any actions that were logged against the atom's retained state */
1629
1630         kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);
1631
1632         if (schedule)
1633                 kbase_js_sched_all(kbdev);
1634
1635         if (!atomic_dec_return(&kctx->work_count)) {
1636                 /* If worker now idle then post all events that jd_done_nolock()
1637                  * has queued */
1638                 mutex_lock(&jctx->lock);
1639                 while (!list_empty(&kctx->completed_jobs)) {
1640                         struct kbase_jd_atom *atom = list_entry(
1641                                         kctx->completed_jobs.next,
1642                                         struct kbase_jd_atom, dep_item[0]);
1643                         list_del(kctx->completed_jobs.next);
1644
1645                         kbase_event_post(kctx, atom);
1646                 }
1647                 mutex_unlock(&jctx->lock);
1648         }
1649
1650         kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity,
1651                         coreref_state);
1652
1653         KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
1654 }
1655
1656 /**
1657  * jd_cancel_worker - Work queue job cancel function.
1658  * @data: a &struct work_struct
1659  *
1660  * Only called as part of 'Zapping' a context (which occurs on termination).
1661  * Operates serially with the kbase_jd_done_worker() on the work queue.
1662  *
1663  * This can only be called on contexts that aren't scheduled.
1664  *
1665  * We don't need to release most of the resources that would occur on
1666  * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be
1667  * running (by virtue of only being called on contexts that aren't
1668  * scheduled).
1669  */
1670 static void jd_cancel_worker(struct work_struct *data)
1671 {
1672         struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
1673         struct kbase_jd_context *jctx;
1674         struct kbase_context *kctx;
1675         struct kbasep_js_kctx_info *js_kctx_info;
1676         bool need_to_try_schedule_context;
1677         bool attr_state_changed;
1678         struct kbase_device *kbdev;
1679
1680         /* Soft jobs should never reach this function */
1681         KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
1682
1683         kctx = katom->kctx;
1684         kbdev = kctx->kbdev;
1685         jctx = &kctx->jctx;
1686         js_kctx_info = &kctx->jctx.sched_info;
1687
1688         KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
1689
1690         /* This only gets called on contexts that are scheduled out. Hence, we must
1691          * make sure we don't de-ref the number of running jobs (there aren't
1692          * any), nor must we try to schedule out the context (it's already
1693          * scheduled out).
1694          */
1695         KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
1696
1697         /* Scheduler: Remove the job from the system */
1698         mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
1699         attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
1700         mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1701
1702         mutex_lock(&jctx->lock);
1703
1704         need_to_try_schedule_context = jd_done_nolock(katom, NULL);
1705         /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
1706          * schedule the context. There's also no need for the jsctx_mutex to have been taken
1707          * around this too. */
1708         KBASE_DEBUG_ASSERT(!need_to_try_schedule_context);
1709
1710         /* katom may have been freed now, do not use! */
1711         mutex_unlock(&jctx->lock);
1712
1713         if (attr_state_changed)
1714                 kbase_js_sched_all(kbdev);
1715 }
1716
1717 /**
1718  * jd_evict_worker - Work queue job evict function
1719  * @data: a &struct work_struct
1720  *
1721  * Only called as part of evicting failed jobs. This is only called on jobs that
1722  * were never submitted to HW Access. Jobs that were submitted are handled
1723  * through kbase_jd_done_worker().
1724  * Operates serially with the kbase_jd_done_worker() on the work queue.
1725  *
1726  * We don't need to release most of the resources that would occur on
1727  * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be
1728  * running (by virtue of having not been submitted to HW Access).
1729  */
1730 static void jd_evict_worker(struct work_struct *data)
1731 {
1732         struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom,
1733                                                                         work);
1734         struct kbase_jd_context *jctx;
1735         struct kbase_context *kctx;
1736         struct kbasep_js_kctx_info *js_kctx_info;
1737         struct kbase_device *kbdev;
1738
1739         /* Soft jobs should never reach this function */
1740         KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
1741
1742         kctx = katom->kctx;
1743         kbdev = kctx->kbdev;
1744         jctx = &kctx->jctx;
1745         js_kctx_info = &kctx->jctx.sched_info;
1746
1747         KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
1748
1749         /* Scheduler: Remove the job from the system */
1750         mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
1751         kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
1752         mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
1753
1754         mutex_lock(&jctx->lock);
1755         jd_done_nolock(katom, NULL);
1756         /* katom may have been freed now, do not use! */
1757         mutex_unlock(&jctx->lock);
1758
1759         kbase_js_sched_all(kbdev);
1760 }
1761
1762 /**
1763  * kbase_jd_done - Complete a job that has been removed from the Hardware
1764  * @katom: atom which has been completed
1765  * @slot_nr: slot the atom was on
1766  * @end_timestamp: completion time
1767  * @done_code: completion code
1768  *
1769  * This must be used whenever a job has been removed from the Hardware, e.g.:
1770  * An IRQ indicates that the job finished (for both error and 'done' codes), or
1771  * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop.
1772  *
1773  * Some work is carried out immediately, and the rest is deferred onto a
1774  * workqueue
1775  *
1776  * Context:
1777  *   This can be called safely from atomic context.
1778  *   The caller must hold kbasep_js_device_data.runpool_irq.lock
1779  */
1780 void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
1781                 ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
1782 {
1783         struct kbase_context *kctx;
1784         struct kbase_device *kbdev;
1785
1786         KBASE_DEBUG_ASSERT(katom);
1787         kctx = katom->kctx;
1788         KBASE_DEBUG_ASSERT(kctx);
1789         kbdev = kctx->kbdev;
1790         KBASE_DEBUG_ASSERT(kbdev);
1791
1792         if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
1793                 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1794
1795         KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0);
1796
1797         kbase_job_check_leave_disjoint(kbdev, katom);
1798
1799         katom->slot_nr = slot_nr;
1800
1801         atomic_inc(&kctx->work_count);
1802
1803 #ifdef CONFIG_DEBUG_FS
1804         /* a failed job happened and is waiting for dumping*/
1805         if (kbase_debug_job_fault_process(katom, katom->event_code))
1806                 return;
1807 #endif
1808
1809         WARN_ON(work_pending(&katom->work));
1810         KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
1811         INIT_WORK(&katom->work, kbase_jd_done_worker);
1812         queue_work(kctx->jctx.job_done_wq, &katom->work);
1813 }
1814
1815 KBASE_EXPORT_TEST_API(kbase_jd_done);
1816
1817 void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
1818 {
1819         struct kbase_context *kctx;
1820         struct kbasep_js_kctx_info *js_kctx_info;
1821
1822         KBASE_DEBUG_ASSERT(NULL != kbdev);
1823         KBASE_DEBUG_ASSERT(NULL != katom);
1824         kctx = katom->kctx;
1825         KBASE_DEBUG_ASSERT(NULL != kctx);
1826
1827         js_kctx_info = &kctx->jctx.sched_info;
1828
1829         KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
1830
1831         /* This should only be done from a context that is not scheduled */
1832         KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
1833
1834         WARN_ON(work_pending(&katom->work));
1835
1836         katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1837
1838         KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
1839         INIT_WORK(&katom->work, jd_cancel_worker);
1840         queue_work(kctx->jctx.job_done_wq, &katom->work);
1841 }
1842
1843 void kbase_jd_evict(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
1844 {
1845         struct kbase_context *kctx;
1846         struct kbasep_js_kctx_info *js_kctx_info;
1847
1848         KBASE_DEBUG_ASSERT(NULL != kbdev);
1849         KBASE_DEBUG_ASSERT(NULL != katom);
1850         kctx = katom->kctx;
1851         KBASE_DEBUG_ASSERT(NULL != kctx);
1852
1853         js_kctx_info = &kctx->jctx.sched_info;
1854
1855         KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
1856
1857         /* This should only be done from a context that is currently scheduled
1858          */
1859         KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
1860
1861         WARN_ON(work_pending(&katom->work));
1862
1863         KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
1864         INIT_WORK(&katom->work, jd_evict_worker);
1865         queue_work(kctx->jctx.job_done_wq, &katom->work);
1866 }
1867
1868 void kbase_jd_zap_context(struct kbase_context *kctx)
1869 {
1870         struct kbase_jd_atom *katom;
1871         struct list_head *entry, *tmp;
1872         struct kbase_device *kbdev;
1873
1874         KBASE_DEBUG_ASSERT(kctx);
1875
1876         kbdev = kctx->kbdev;
1877
1878         KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
1879
1880         kbase_js_zap_context(kctx);
1881
1882         mutex_lock(&kctx->jctx.lock);
1883
1884         /*
1885          * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are
1886          * queued outside the job scheduler.
1887          */
1888
1889         list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) {
1890                 katom = list_entry(entry, struct kbase_jd_atom, dep_item[0]);
1891                 kbase_cancel_soft_job(katom);
1892         }
1893
1894
1895 #ifdef CONFIG_KDS
1896
1897         /* For each job waiting on a kds resource, cancel the wait and force the job to
1898          * complete early, this is done so that we don't leave jobs outstanding waiting
1899          * on kds resources which may never be released when contexts are zapped, resulting
1900          * in a hang.
1901          *
1902          * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held,
1903          * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job.
1904          */
1905
1906         list_for_each(entry, &kctx->waiting_kds_resource) {
1907                 katom = list_entry(entry, struct kbase_jd_atom, node);
1908
1909                 kbase_cancel_kds_wait_job(katom);
1910         }
1911 #endif
1912
1913         mutex_unlock(&kctx->jctx.lock);
1914
1915         kbase_jm_wait_for_zero_jobs(kctx);
1916 }
1917
1918 KBASE_EXPORT_TEST_API(kbase_jd_zap_context);
1919
1920 int kbase_jd_init(struct kbase_context *kctx)
1921 {
1922         int i;
1923         int mali_err = 0;
1924 #ifdef CONFIG_KDS
1925         int err;
1926 #endif                          /* CONFIG_KDS */
1927
1928         KBASE_DEBUG_ASSERT(kctx);
1929
1930         kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", 0, 1);
1931         if (NULL == kctx->jctx.job_done_wq) {
1932                 mali_err = -ENOMEM;
1933                 goto out1;
1934         }
1935
1936         for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
1937                 init_waitqueue_head(&kctx->jctx.atoms[i].completed);
1938
1939                 INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]);
1940                 INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]);
1941
1942                 /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */
1943                 kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
1944                 kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
1945         }
1946
1947         mutex_init(&kctx->jctx.lock);
1948
1949         init_waitqueue_head(&kctx->jctx.zero_jobs_wait);
1950
1951         spin_lock_init(&kctx->jctx.tb_lock);
1952
1953 #ifdef CONFIG_KDS
1954         err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear);
1955         if (0 != err) {
1956                 mali_err = -EINVAL;
1957                 goto out2;
1958         }
1959 #endif                          /* CONFIG_KDS */
1960
1961         kctx->jctx.job_nr = 0;
1962         INIT_LIST_HEAD(&kctx->completed_jobs);
1963         atomic_set(&kctx->work_count, 0);
1964
1965         return 0;
1966
1967 #ifdef CONFIG_KDS
1968  out2:
1969         destroy_workqueue(kctx->jctx.job_done_wq);
1970 #endif                          /* CONFIG_KDS */
1971  out1:
1972         return mali_err;
1973 }
1974
1975 KBASE_EXPORT_TEST_API(kbase_jd_init);
1976
1977 void kbase_jd_exit(struct kbase_context *kctx)
1978 {
1979         KBASE_DEBUG_ASSERT(kctx);
1980
1981 #ifdef CONFIG_KDS
1982         kds_callback_term(&kctx->jctx.kds_cb);
1983 #endif                          /* CONFIG_KDS */
1984         /* Work queue is emptied by this */
1985         destroy_workqueue(kctx->jctx.job_done_wq);
1986 }
1987
1988 KBASE_EXPORT_TEST_API(kbase_jd_exit);