drivers/gpu/arm/mali400/mali/common/mali_soft_job.c

   1 /*
   2  * Copyright (C) 2013-2014, 2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_soft_job.h"
  12 #include "mali_osk.h"
  13 #include "mali_timeline.h"
  14 #include "mali_session.h"
  15 #include "mali_kernel_common.h"
  16 #include "mali_uk_types.h"
  17 #include "mali_scheduler.h"
  18 #include "mali_executor.h"
  19
  20 MALI_STATIC_INLINE void mali_soft_job_system_lock(struct mali_soft_job_system *system)
  21 {
  22         MALI_DEBUG_ASSERT_POINTER(system);
  23         _mali_osk_spinlock_irq_lock(system->lock);
  24         MALI_DEBUG_PRINT(5, ("Mali Soft Job: soft system %p lock taken\n", system));
  25         MALI_DEBUG_ASSERT(0 == system->lock_owner);
  26         MALI_DEBUG_CODE(system->lock_owner = _mali_osk_get_tid());
  27 }
  28
  29 MALI_STATIC_INLINE void mali_soft_job_system_unlock(struct mali_soft_job_system *system)
  30 {
  31         MALI_DEBUG_ASSERT_POINTER(system);
  32         MALI_DEBUG_PRINT(5, ("Mali Soft Job: releasing soft system %p lock\n", system));
  33         MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner);
  34         MALI_DEBUG_CODE(system->lock_owner = 0);
  35         _mali_osk_spinlock_irq_unlock(system->lock);
  36 }
  37
  38 #if defined(DEBUG)
  39 MALI_STATIC_INLINE void mali_soft_job_system_assert_locked(struct mali_soft_job_system *system)
  40 {
  41         MALI_DEBUG_ASSERT_POINTER(system);
  42         MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner);
  43 }
  44 #define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) mali_soft_job_system_assert_locked(system)
  45 #else
  46 #define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system)
  47 #endif /* defined(DEBUG) */
  48
  49 struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session)
  50 {
  51         struct mali_soft_job_system *system;
  52
  53         MALI_DEBUG_ASSERT_POINTER(session);
  54
  55         system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system));
  56         if (NULL == system) {
  57                 return NULL;
  58         }
  59
  60         system->session = session;
  61
  62         system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER);
  63         if (NULL == system->lock) {
  64                 mali_soft_job_system_destroy(system);
  65                 return NULL;
  66         }
  67         system->lock_owner = 0;
  68         system->last_job_id = 0;
  69
  70         _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used));
  71
  72         return system;
  73 }
  74
  75 void mali_soft_job_system_destroy(struct mali_soft_job_system *system)
  76 {
  77         MALI_DEBUG_ASSERT_POINTER(system);
  78
  79         /* All jobs should be free at this point. */
  80         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&(system->jobs_used)));
  81
  82         if (NULL != system) {
  83                 if (NULL != system->lock) {
  84                         _mali_osk_spinlock_irq_term(system->lock);
  85                 }
  86                 _mali_osk_free(system);
  87         }
  88 }
  89
  90 static void mali_soft_job_system_free_job(struct mali_soft_job_system *system, struct mali_soft_job *job)
  91 {
  92         MALI_DEBUG_ASSERT_POINTER(job);
  93         MALI_DEBUG_ASSERT_POINTER(system);
  94
  95         mali_soft_job_system_lock(job->system);
  96
  97         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id);
  98         MALI_DEBUG_ASSERT(system == job->system);
  99
 100         _mali_osk_list_del(&(job->system_list));
 101
 102         mali_soft_job_system_unlock(job->system);
 103
 104         _mali_osk_free(job);
 105 }
 106
 107 MALI_STATIC_INLINE struct mali_soft_job *mali_soft_job_system_lookup_job(struct mali_soft_job_system *system, u32 job_id)
 108 {
 109         struct mali_soft_job *job, *tmp;
 110
 111         MALI_DEBUG_ASSERT_POINTER(system);
 112         MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system);
 113
 114         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) {
 115                 if (job->id == job_id)
 116                         return job;
 117         }
 118
 119         return NULL;
 120 }
 121
 122 void mali_soft_job_destroy(struct mali_soft_job *job)
 123 {
 124         MALI_DEBUG_ASSERT_POINTER(job);
 125         MALI_DEBUG_ASSERT_POINTER(job->system);
 126
 127         MALI_DEBUG_PRINT(4, ("Mali Soft Job: destroying soft job %u (0x%08X)\n", job->id, job));
 128
 129         if (NULL != job) {
 130                 if (0 < _mali_osk_atomic_dec_return(&job->refcount)) return;
 131
 132                 _mali_osk_atomic_term(&job->refcount);
 133
 134                 if (NULL != job->activated_notification) {
 135                         _mali_osk_notification_delete(job->activated_notification);
 136                         job->activated_notification = NULL;
 137                 }
 138
 139                 mali_soft_job_system_free_job(job->system, job);
 140         }
 141 }
 142
 143 struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job)
 144 {
 145         struct mali_soft_job *job;
 146         _mali_osk_notification_t *notification = NULL;
 147
 148         MALI_DEBUG_ASSERT_POINTER(system);
 149         MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) ||
 150                           (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type));
 151
 152         notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s));
 153         if (unlikely(NULL == notification)) {
 154                 MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification"));
 155                 return NULL;
 156         }
 157
 158         job = _mali_osk_malloc(sizeof(struct mali_soft_job));
 159         if (unlikely(NULL == job)) {
 160                 MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n"));
 161                 return NULL;
 162         }
 163
 164         mali_soft_job_system_lock(system);
 165
 166         job->system = system;
 167         job->id = system->last_job_id++;
 168         job->state = MALI_SOFT_JOB_STATE_ALLOCATED;
 169
 170         _mali_osk_list_add(&(job->system_list), &(system->jobs_used));
 171
 172         job->type = type;
 173         job->user_job = user_job;
 174         job->activated = MALI_FALSE;
 175
 176         job->activated_notification = notification;
 177
 178         _mali_osk_atomic_init(&job->refcount, 1);
 179
 180         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
 181         MALI_DEBUG_ASSERT(system == job->system);
 182         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id);
 183
 184         mali_soft_job_system_unlock(system);
 185
 186         return job;
 187 }
 188
 189 mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence)
 190 {
 191         mali_timeline_point point;
 192         struct mali_soft_job_system *system;
 193
 194         MALI_DEBUG_ASSERT_POINTER(job);
 195         MALI_DEBUG_ASSERT_POINTER(fence);
 196
 197         MALI_DEBUG_ASSERT_POINTER(job->system);
 198         system = job->system;
 199
 200         MALI_DEBUG_ASSERT_POINTER(system->session);
 201         MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system);
 202
 203         mali_soft_job_system_lock(system);
 204
 205         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
 206         job->state = MALI_SOFT_JOB_STATE_STARTED;
 207
 208         mali_soft_job_system_unlock(system);
 209
 210         MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job));
 211
 212         mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job);
 213         point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT);
 214
 215         return point;
 216 }
 217
 218 static mali_bool mali_soft_job_is_activated(void *data)
 219 {
 220         struct mali_soft_job *job;
 221
 222         job = (struct mali_soft_job *) data;
 223         MALI_DEBUG_ASSERT_POINTER(job);
 224
 225         return job->activated;
 226 }
 227
 228 _mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id)
 229 {
 230         struct mali_soft_job *job;
 231         struct mali_timeline_system *timeline_system;
 232         mali_scheduler_mask schedule_mask;
 233
 234         MALI_DEBUG_ASSERT_POINTER(system);
 235
 236         mali_soft_job_system_lock(system);
 237
 238         job = mali_soft_job_system_lookup_job(system, job_id);
 239
 240         if ((NULL == job) || (MALI_SOFT_JOB_TYPE_USER_SIGNALED != job->type)
 241             || !(MALI_SOFT_JOB_STATE_STARTED == job->state || MALI_SOFT_JOB_STATE_TIMED_OUT == job->state)) {
 242                 mali_soft_job_system_unlock(system);
 243                 MALI_PRINT_ERROR(("Mali Soft Job: invalid soft job id %u", job_id));
 244                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 245         }
 246
 247         if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) {
 248                 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
 249                 mali_soft_job_system_unlock(system);
 250
 251                 MALI_DEBUG_ASSERT(MALI_TRUE == job->activated);
 252                 MALI_DEBUG_PRINT(4, ("Mali Soft Job: soft job %u (0x%08X) was timed out\n", job->id, job));
 253                 mali_soft_job_destroy(job);
 254
 255                 return _MALI_OSK_ERR_TIMEOUT;
 256         }
 257
 258         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
 259
 260         job->state = MALI_SOFT_JOB_STATE_SIGNALED;
 261         mali_soft_job_system_unlock(system);
 262
 263         /* Since the job now is in signaled state, timeouts from the timeline system will be
 264          * ignored, and it is not possible to signal this job again. */
 265
 266         timeline_system = system->session->timeline_system;
 267         MALI_DEBUG_ASSERT_POINTER(timeline_system);
 268
 269         /* Wait until activated. */
 270         _mali_osk_wait_queue_wait_event(timeline_system->wait_queue, mali_soft_job_is_activated, (void *) job);
 271
 272         MALI_DEBUG_PRINT(4, ("Mali Soft Job: signaling soft job %u (0x%08X)\n", job->id, job));
 273
 274         schedule_mask = mali_timeline_tracker_release(&job->tracker);
 275         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
 276
 277         mali_soft_job_destroy(job);
 278
 279         return _MALI_OSK_ERR_OK;
 280 }
 281
 282 static void mali_soft_job_send_activated_notification(struct mali_soft_job *job)
 283 {
 284         if (NULL != job->activated_notification) {
 285                 _mali_uk_soft_job_activated_s *res = job->activated_notification->result_buffer;
 286                 res->user_job = job->user_job;
 287                 mali_session_send_notification(job->system->session, job->activated_notification);
 288         }
 289         job->activated_notification = NULL;
 290 }
 291
 292 mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job)
 293 {
 294         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 295
 296         MALI_DEBUG_ASSERT_POINTER(job);
 297         MALI_DEBUG_ASSERT_POINTER(job->system);
 298         MALI_DEBUG_ASSERT_POINTER(job->system->session);
 299
 300         MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline activation for soft job %u (0x%08X).\n", job->id, job));
 301
 302         mali_soft_job_system_lock(job->system);
 303
 304         if (unlikely(job->system->session->is_aborting)) {
 305                 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Soft job %u (0x%08X) activated while session is aborting.\n", job->id, job));
 306
 307                 mali_soft_job_system_unlock(job->system);
 308
 309                 /* Since we are in shutdown, we can ignore the scheduling bitmask. */
 310                 mali_timeline_tracker_release(&job->tracker);
 311                 mali_soft_job_destroy(job);
 312                 return schedule_mask;
 313         }
 314
 315         /* Send activated notification. */
 316         mali_soft_job_send_activated_notification(job);
 317
 318         /* Wake up sleeping signaler. */
 319         job->activated = MALI_TRUE;
 320
 321         /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */
 322         if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) {
 323                 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
 324
 325                 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
 326                 mali_soft_job_system_unlock(job->system);
 327
 328                 schedule_mask |= mali_timeline_tracker_release(&job->tracker);
 329
 330                 mali_soft_job_destroy(job);
 331         } else {
 332                 _mali_osk_wait_queue_wake_up(job->tracker.system->wait_queue);
 333
 334                 mali_soft_job_system_unlock(job->system);
 335         }
 336
 337         return schedule_mask;
 338 }
 339
 340 mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job)
 341 {
 342         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 343
 344         MALI_DEBUG_ASSERT_POINTER(job);
 345         MALI_DEBUG_ASSERT_POINTER(job->system);
 346         MALI_DEBUG_ASSERT_POINTER(job->system->session);
 347         MALI_DEBUG_ASSERT(MALI_TRUE == job->activated);
 348
 349         MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline timeout for soft job %u (0x%08X).\n", job->id, job));
 350
 351         mali_soft_job_system_lock(job->system);
 352
 353         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED  == job->state ||
 354                           MALI_SOFT_JOB_STATE_SIGNALED == job->state);
 355
 356         if (unlikely(job->system->session->is_aborting)) {
 357                 /* The session is aborting.  This job will be released and destroyed by @ref
 358                  * mali_soft_job_system_abort(). */
 359                 mali_soft_job_system_unlock(job->system);
 360
 361                 return MALI_SCHEDULER_MASK_EMPTY;
 362         }
 363
 364         if (MALI_SOFT_JOB_STATE_STARTED != job->state) {
 365                 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state);
 366
 367                 /* The job is about to be signaled, ignore timeout. */
 368                 MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeout on soft job %u (0x%08X) in signaled state.\n", job->id, job));
 369                 mali_soft_job_system_unlock(job->system);
 370                 return schedule_mask;
 371         }
 372
 373         MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
 374
 375         job->state = MALI_SOFT_JOB_STATE_TIMED_OUT;
 376         _mali_osk_atomic_inc(&job->refcount);
 377
 378         mali_soft_job_system_unlock(job->system);
 379
 380         schedule_mask = mali_timeline_tracker_release(&job->tracker);
 381
 382         mali_soft_job_destroy(job);
 383
 384         return schedule_mask;
 385 }
 386
 387 void mali_soft_job_system_abort(struct mali_soft_job_system *system)
 388 {
 389         struct mali_soft_job *job, *tmp;
 390         _MALI_OSK_LIST_HEAD_STATIC_INIT(jobs);
 391
 392         MALI_DEBUG_ASSERT_POINTER(system);
 393         MALI_DEBUG_ASSERT_POINTER(system->session);
 394         MALI_DEBUG_ASSERT(system->session->is_aborting);
 395
 396         MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting soft job system for session 0x%08X.\n", system->session));
 397
 398         mali_soft_job_system_lock(system);
 399
 400         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) {
 401                 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED   == job->state ||
 402                                   MALI_SOFT_JOB_STATE_TIMED_OUT == job->state);
 403
 404                 if (MALI_SOFT_JOB_STATE_STARTED == job->state) {
 405                         /* If the job has been activated, we have to release the tracker and destroy
 406                          * the job.  If not, the tracker will be released and the job destroyed when
 407                          * it is activated. */
 408                         if (MALI_TRUE == job->activated) {
 409                                 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting unsignaled soft job %u (0x%08X).\n", job->id, job));
 410
 411                                 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
 412                                 _mali_osk_list_move(&job->system_list, &jobs);
 413                         }
 414                 } else if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) {
 415                         MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting timed out soft job %u (0x%08X).\n", job->id, job));
 416
 417                         /* We need to destroy this soft job. */
 418                         _mali_osk_list_move(&job->system_list, &jobs);
 419                 }
 420         }
 421
 422         mali_soft_job_system_unlock(system);
 423
 424         /* Release and destroy jobs. */
 425         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &jobs, struct mali_soft_job, system_list) {
 426                 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED  == job->state ||
 427                                   MALI_SOFT_JOB_STATE_TIMED_OUT == job->state);
 428
 429                 if (MALI_SOFT_JOB_STATE_SIGNALED == job->state) {
 430                         mali_timeline_tracker_release(&job->tracker);
 431                 }
 432
 433                 /* Move job back to used list before destroying. */
 434                 _mali_osk_list_move(&job->system_list, &system->jobs_used);
 435
 436                 mali_soft_job_destroy(job);
 437         }
 438 }