drivers/gpu/arm/midgard/mali_kbase_softjobs.c

   1 /*
   2  *
   3  * (C) COPYRIGHT ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 #include <mali_kbase.h>
  21
  22 #ifdef CONFIG_SYNC
  23 #include "sync.h"
  24 #include <linux/syscalls.h>
  25 #include "mali_kbase_sync.h"
  26 #endif
  27
  28
  29 /* Mask to check cache alignment of data structures */
  30 #define KBASE_CACHE_ALIGNMENT_MASK              ((1<<L1_CACHE_SHIFT)-1)
  31
  32 /**
  33  * @file mali_kbase_softjobs.c
  34  *
  35  * This file implements the logic behind software only jobs that are
  36  * executed within the driver rather than being handed over to the GPU.
  37  */
  38
  39 static int kbase_dump_cpu_gpu_time(kbase_jd_atom *katom)
  40 {
  41         kbase_va_region *reg;
  42         phys_addr_t addr = 0;
  43         u64 pfn;
  44         u32 offset;
  45         char *page;
  46         struct timespec ts;
  47         base_dump_cpu_gpu_counters data;
  48         u64 system_time;
  49         u64 cycle_counter;
  50         mali_addr64 jc = katom->jc;
  51         kbase_context *kctx = katom->kctx;
  52         int pm_active_err;
  53
  54         u32 hi1, hi2;
  55
  56         memset(&data, 0, sizeof(data));
  57
  58         /* Take the PM active reference as late as possible - otherwise, it could
  59          * delay suspend until we process the atom (which may be at the end of a
  60          * long chain of dependencies */
  61         pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE);
  62         if (pm_active_err) {
  63                 kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data;
  64
  65                 /* We're suspended - queue this on the list of suspended jobs
  66                  * Use dep_item[1], because dep_item[0] is in use for 'waiting_soft_jobs' */
  67                 mutex_lock(&js_devdata->runpool_mutex);
  68                 list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list);
  69                 mutex_unlock(&js_devdata->runpool_mutex);
  70
  71                 return pm_active_err;
  72         }
  73
  74         kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
  75
  76         /* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
  77         do {
  78                 hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
  79                 cycle_counter = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
  80                 hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
  81                 cycle_counter |= (((u64) hi1) << 32);
  82         } while (hi1 != hi2);
  83
  84         /* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
  85         do {
  86                 hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
  87                 system_time = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
  88                 hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
  89                 system_time |= (((u64) hi1) << 32);
  90         } while (hi1 != hi2);
  91
  92         /* Record the CPU's idea of current time */
  93         getnstimeofday(&ts);
  94
  95         kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
  96
  97         kbase_pm_context_idle(kctx->kbdev);
  98
  99         data.sec = ts.tv_sec;
 100         data.usec = ts.tv_nsec / 1000;
 101         data.system_time = system_time;
 102         data.cycle_counter = cycle_counter;
 103
 104         pfn = jc >> PAGE_SHIFT;
 105         offset = jc & ~PAGE_MASK;
 106
 107         /* Assume this atom will be cancelled until we know otherwise */
 108         katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 109         if (offset > 0x1000 - sizeof(data)) {
 110                 /* Wouldn't fit in the page */
 111                 return 0;
 112         }
 113
 114         kbase_gpu_vm_lock(kctx);
 115         reg = kbase_region_tracker_find_region_enclosing_address(kctx, jc);
 116         if (reg &&
 117             (reg->flags & KBASE_REG_GPU_WR) &&
 118             reg->alloc && reg->alloc->pages)
 119                 addr = reg->alloc->pages[pfn - reg->start_pfn];
 120
 121         kbase_gpu_vm_unlock(kctx);
 122         if (!addr)
 123                 return 0;
 124
 125         page = kmap(pfn_to_page(PFN_DOWN(addr)));
 126         if (!page)
 127                 return 0;
 128
 129         memcpy(page + offset, &data, sizeof(data));
 130         kbase_sync_to_cpu(addr + offset, page + offset, sizeof(data));
 131         kunmap(pfn_to_page(PFN_DOWN(addr)));
 132
 133         /* Atom was fine - mark it as done */
 134         katom->event_code = BASE_JD_EVENT_DONE;
 135
 136         return 0;
 137 }
 138
 139 #ifdef CONFIG_SYNC
 140
 141 /* Complete an atom that has returned '1' from kbase_process_soft_job (i.e. has waited)
 142  *
 143  * @param katom     The atom to complete
 144  */
 145 static void complete_soft_job(kbase_jd_atom *katom)
 146 {
 147         kbase_context *kctx = katom->kctx;
 148
 149         mutex_lock(&kctx->jctx.lock);
 150         list_del(&katom->dep_item[0]);
 151         kbase_finish_soft_job(katom);
 152         if (jd_done_nolock(katom))
 153                 kbasep_js_try_schedule_head_ctx(kctx->kbdev);
 154         mutex_unlock(&kctx->jctx.lock);
 155 }
 156
 157 static base_jd_event_code kbase_fence_trigger(kbase_jd_atom *katom, int result)
 158 {
 159         struct sync_pt *pt;
 160         struct sync_timeline *timeline;
 161
 162         if (!list_is_singular(&katom->fence->pt_list_head)) {
 163                 /* Not exactly one item in the list - so it didn't (directly) come from us */
 164                 return BASE_JD_EVENT_JOB_CANCELLED;
 165         }
 166
 167         pt = list_first_entry(&katom->fence->pt_list_head, struct sync_pt, pt_list);
 168         timeline = pt->parent;
 169
 170         if (!kbase_sync_timeline_is_ours(timeline)) {
 171                 /* Fence has a sync_pt which isn't ours! */
 172                 return BASE_JD_EVENT_JOB_CANCELLED;
 173         }
 174
 175         kbase_sync_signal_pt(pt, result);
 176
 177         sync_timeline_signal(timeline);
 178
 179         return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE;
 180 }
 181
 182 static void kbase_fence_wait_worker(struct work_struct *data)
 183 {
 184         kbase_jd_atom *katom;
 185         kbase_context *kctx;
 186
 187         katom = container_of(data, kbase_jd_atom, work);
 188         kctx = katom->kctx;
 189
 190         complete_soft_job(katom);
 191 }
 192
 193 static void kbase_fence_wait_callback(struct sync_fence *fence, struct sync_fence_waiter *waiter)
 194 {
 195         kbase_jd_atom *katom = container_of(waiter, kbase_jd_atom, sync_waiter);
 196         kbase_context *kctx;
 197
 198         KBASE_DEBUG_ASSERT(NULL != katom);
 199
 200         kctx = katom->kctx;
 201
 202         KBASE_DEBUG_ASSERT(NULL != kctx);
 203
 204         /* Propagate the fence status to the atom.
 205          * If negative then cancel this atom and its dependencies.
 206          */
 207         if (fence->status < 0)
 208         {
 209                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 210         }
 211
 212         /* To prevent a potential deadlock we schedule the work onto the job_done_wq workqueue
 213          *
 214          * The issue is that we may signal the timeline while holding kctx->jctx.lock and
 215          * the callbacks are run synchronously from sync_timeline_signal. So we simply defer the work.
 216          */
 217
 218         KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
 219         INIT_WORK(&katom->work, kbase_fence_wait_worker);
 220         queue_work(kctx->jctx.job_done_wq, &katom->work);
 221 }
 222
 223 static int kbase_fence_wait(kbase_jd_atom *katom)
 224 {
 225         int ret;
 226
 227         KBASE_DEBUG_ASSERT(NULL != katom);
 228         KBASE_DEBUG_ASSERT(NULL != katom->kctx);
 229
 230         sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback);
 231
 232         ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter);
 233
 234         if (ret == 1) {
 235                 /* Already signalled */
 236                 return 0;
 237         } else if (ret < 0) {
 238                 goto cancel_atom;
 239         }
 240         return 1;
 241
 242  cancel_atom:
 243         katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 244         /* We should cause the dependant jobs in the bag to be failed,
 245          * to do this we schedule the work queue to complete this job */
 246         KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
 247         INIT_WORK(&katom->work, kbase_fence_wait_worker);
 248         queue_work(katom->kctx->jctx.job_done_wq, &katom->work);
 249         return 1;
 250 }
 251
 252 static void kbase_fence_cancel_wait(kbase_jd_atom *katom)
 253 {
 254         if(!katom)
 255         {
 256                 pr_err("katom null.forbiden return\n");
 257                 return;
 258         }
 259         if(!katom->fence)
 260         {
 261                 pr_info("katom->fence null.may release out of order.so continue unfinished step\n");
 262                 /*
 263                 if return here,may result in  infinite loop?
 264                 we need to delete dep_item[0] from kctx->waiting_soft_jobs?
 265                 jd_done_nolock function move the dep_item[0] to complete job list and then delete?
 266                 */
 267                 goto finish_softjob;
 268         }
 269         if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0)
 270         {
 271                 /* The wait wasn't cancelled - leave the cleanup for kbase_fence_wait_callback */
 272                 return;
 273         }
 274
 275         /* Wait was cancelled - zap the atoms */
 276 finish_softjob:
 277         katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
 278
 279         kbase_finish_soft_job(katom);
 280
 281         if (jd_done_nolock(katom))
 282                 kbasep_js_try_schedule_head_ctx(katom->kctx->kbdev);
 283         return;
 284
 285 }
 286 #endif /* CONFIG_SYNC */
 287
 288 int kbase_process_soft_job(kbase_jd_atom *katom)
 289 {
 290         int status;
 291         switch (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) {
 292         case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
 293                 return kbase_dump_cpu_gpu_time(katom);
 294 #ifdef CONFIG_SYNC
 295         case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
 296                 KBASE_DEBUG_ASSERT(katom->fence != NULL);
 297                 katom->event_code = kbase_fence_trigger(katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT);
 298                 /* Release the reference as we don't need it any more */
 299                 sync_fence_put(katom->fence);
 300                 katom->fence = NULL;
 301                 break;
 302         case BASE_JD_REQ_SOFT_FENCE_WAIT:
 303                 return kbase_fence_wait(katom);
 304 #endif                          /* CONFIG_SYNC */
 305         case BASE_JD_REQ_SOFT_REPLAY:
 306                 status = kbase_replay_process(katom);
 307                 if (status & MALI_REPLAY_FLAG_JS_RESCHED)
 308                         pr_err("replay called from kbase_process_soft_job - missing resched!\n");
 309                 return status & MALI_REPLAY_STATUS_MASK;
 310         }
 311
 312         /* Atom is complete */
 313         return 0;
 314 }
 315
 316 void kbase_cancel_soft_job(kbase_jd_atom *katom)
 317 {
 318         switch (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) {
 319 #ifdef CONFIG_SYNC
 320         case BASE_JD_REQ_SOFT_FENCE_WAIT:
 321                 kbase_fence_cancel_wait(katom);
 322                 break;
 323 #endif
 324         default:
 325                 /* This soft-job doesn't support cancellation! */
 326                 KBASE_DEBUG_ASSERT(0);
 327         }
 328 }
 329
 330 mali_error kbase_prepare_soft_job(kbase_jd_atom *katom)
 331 {
 332         switch (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) {
 333         case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
 334                 {
 335                         if(0 != (katom->jc & KBASE_CACHE_ALIGNMENT_MASK))
 336                                 return MALI_ERROR_FUNCTION_FAILED;
 337                 }
 338                 break;
 339 #ifdef CONFIG_SYNC
 340         case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
 341                 {
 342                         base_fence fence;
 343                         int fd;
 344                         if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
 345                                 return MALI_ERROR_FUNCTION_FAILED;
 346
 347                         fd = kbase_stream_create_fence(fence.basep.stream_fd);
 348                         if (fd < 0)
 349                                 return MALI_ERROR_FUNCTION_FAILED;
 350
 351                         katom->fence = sync_fence_fdget(fd);
 352
 353                         if (katom->fence == NULL) {
 354                                 /* The only way the fence can be NULL is if userspace closed it for us.
 355                                  * So we don't need to clear it up */
 356                                 return MALI_ERROR_FUNCTION_FAILED;
 357                         }
 358                         fence.basep.fd = fd;
 359                         if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) {
 360                                 katom->fence = NULL;
 361                                 sys_close(fd);
 362                                 return MALI_ERROR_FUNCTION_FAILED;
 363                         }
 364                 }
 365                 break;
 366         case BASE_JD_REQ_SOFT_FENCE_WAIT:
 367                 {
 368                         base_fence fence;
 369                         if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
 370                                 return MALI_ERROR_FUNCTION_FAILED;
 371
 372                         /* Get a reference to the fence object */
 373                         katom->fence = sync_fence_fdget(fence.basep.fd);
 374                         if (katom->fence == NULL)
 375                                 return MALI_ERROR_FUNCTION_FAILED;
 376                 }
 377                 break;
 378 #endif                          /* CONFIG_SYNC */
 379         case BASE_JD_REQ_SOFT_REPLAY:
 380                 break;
 381         default:
 382                 /* Unsupported soft-job */
 383                 return MALI_ERROR_FUNCTION_FAILED;
 384         }
 385         return MALI_ERROR_NONE;
 386 }
 387
 388 void kbase_finish_soft_job(kbase_jd_atom *katom)
 389 {
 390         switch (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) {
 391         case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
 392                 /* Nothing to do */
 393                 break;
 394 #ifdef CONFIG_SYNC
 395         case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
 396                 if (katom->fence) {
 397                         /* The fence has not yet been signalled, so we do it now */
 398                         kbase_fence_trigger(katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT);
 399                         sync_fence_put(katom->fence);
 400                         katom->fence = NULL;
 401                 }
 402                 break;
 403         case BASE_JD_REQ_SOFT_FENCE_WAIT:
 404                 /* Release the reference to the fence object */
 405                 if(katom->fence){
 406                         sync_fence_put(katom->fence);
 407                         katom->fence = NULL;
 408                 }
 409                 break;
 410 #endif                          /* CONFIG_SYNC */
 411         }
 412 }
 413
 414 void kbase_resume_suspended_soft_jobs(kbase_device *kbdev)
 415 {
 416         LIST_HEAD(local_suspended_soft_jobs);
 417         kbase_jd_atom *tmp_iter;
 418         kbase_jd_atom *katom_iter;
 419         kbasep_js_device_data *js_devdata;
 420         mali_bool resched = MALI_FALSE;
 421         KBASE_DEBUG_ASSERT(kbdev);
 422
 423         js_devdata = &kbdev->js_data;
 424
 425         /* Move out the entire list */
 426         mutex_lock(&js_devdata->runpool_mutex);
 427         list_splice_init(&js_devdata->suspended_soft_jobs_list, &local_suspended_soft_jobs);
 428         mutex_unlock(&js_devdata->runpool_mutex);
 429
 430         /* Each atom must be detached from the list and ran separately - it could
 431          * be re-added to the old list, but this is unlikely */
 432         list_for_each_entry_safe(katom_iter, tmp_iter, &local_suspended_soft_jobs, dep_item[1])
 433         {
 434                 kbase_context *kctx = katom_iter->kctx;
 435                 mutex_lock(&kctx->jctx.lock);
 436
 437                 /* Remove from the global list */
 438                 list_del(&katom_iter->dep_item[1]);
 439                 /* Remove from the context's list of waiting soft jobs */
 440                 list_del(&katom_iter->dep_item[0]);
 441
 442                 if (kbase_process_soft_job(katom_iter) == 0) {
 443                         kbase_finish_soft_job(katom_iter);
 444                         resched |= jd_done_nolock(katom_iter);
 445                 } else {
 446                         /* The job has not completed */
 447                         KBASE_DEBUG_ASSERT((katom_iter->core_req & BASEP_JD_REQ_ATOM_TYPE)
 448                                                                 != BASE_JD_REQ_SOFT_REPLAY);
 449                         list_add_tail(&katom_iter->dep_item[0], &kctx->waiting_soft_jobs);
 450                 }
 451
 452                 mutex_unlock(&kctx->jctx.lock);
 453         }
 454
 455         if (resched)
 456                 kbasep_js_try_schedule_head_ctx(kbdev);
 457 }