drivers/gpu/arm/mali400/mali/common/mali_timeline.c

   1 /*
   2  * Copyright (C) 2013-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_timeline.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_scheduler.h"
  14 #include "mali_soft_job.h"
  15 #include "mali_timeline_fence_wait.h"
  16 #include "mali_timeline_sync_fence.h"
  17 #include "mali_executor.h"
  18 #include "mali_pp_job.h"
  19
  20 #define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
  21
  22 /*
  23  * Following three elements are used to record how many
  24  * gp, physical pp or virtual pp jobs are delayed in the whole
  25  * timeline system, we can use these three value to decide
  26  * if need to deactivate idle group.
  27  */
  28 _mali_osk_atomic_t gp_tracker_count;
  29 _mali_osk_atomic_t phy_pp_tracker_count;
  30 _mali_osk_atomic_t virt_pp_tracker_count;
  31
  32 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
  33                 struct mali_timeline_waiter *waiter);
  34
  35 #if defined(CONFIG_SYNC)
  36 #include <linux/version.h>
  37 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
  38 #include <linux/list.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/spinlock.h>
  41
  42 struct mali_deferred_fence_put_entry {
  43         struct hlist_node list;
  44         struct sync_fence *fence;
  45 };
  46
  47 static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
  48 static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
  49
  50 static void put_sync_fences(struct work_struct *ignore)
  51 {
  52         struct hlist_head list;
  53         struct hlist_node *tmp, *pos;
  54         unsigned long flags;
  55         struct mali_deferred_fence_put_entry *o;
  56
  57         spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
  58         hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
  59         spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
  60
  61         hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
  62                 sync_fence_put(o->fence);
  63                 kfree(o);
  64         }
  65 }
  66
  67 static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
  68 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
  69
  70 /* Callback that is called when a sync fence a tracker is waiting on is signaled. */
  71 static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
  72 {
  73         struct mali_timeline_system  *system;
  74         struct mali_timeline_waiter  *waiter;
  75         struct mali_timeline_tracker *tracker;
  76         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  77         u32 tid = _mali_osk_get_tid();
  78         mali_bool is_aborting = MALI_FALSE;
  79 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
  80         int fence_status = sync_fence->status;
  81 #else
  82         int fence_status = atomic_read(&sync_fence->status);
  83 #endif
  84
  85         MALI_DEBUG_ASSERT_POINTER(sync_fence);
  86         MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
  87
  88         tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
  89         MALI_DEBUG_ASSERT_POINTER(tracker);
  90
  91         system = tracker->system;
  92         MALI_DEBUG_ASSERT_POINTER(system);
  93         MALI_DEBUG_ASSERT_POINTER(system->session);
  94
  95         mali_spinlock_reentrant_wait(system->spinlock, tid);
  96
  97         is_aborting = system->session->is_aborting;
  98         if (!is_aborting && (0 > fence_status)) {
  99                 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
 100                 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
 101         }
 102
 103         waiter = tracker->waiter_sync;
 104         MALI_DEBUG_ASSERT_POINTER(waiter);
 105
 106         tracker->sync_fence = NULL;
 107         tracker->fence.sync_fd = -1;
 108
 109         schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
 110
 111         /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
 112         if (is_aborting) {
 113                 _mali_osk_wait_queue_wake_up(system->wait_queue);
 114         }
 115
 116         mali_spinlock_reentrant_signal(system->spinlock, tid);
 117
 118         /*
 119          * Older versions of Linux, before 3.5, doesn't support fput() in interrupt
 120          * context. For those older kernels, allocate a list object and put the
 121          * fence object on that and defer the call to sync_fence_put() to a workqueue.
 122          */
 123 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
 124         {
 125                 struct mali_deferred_fence_put_entry *obj;
 126
 127                 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
 128                 if (obj) {
 129                         unsigned long flags;
 130                         mali_bool schedule = MALI_FALSE;
 131
 132                         obj->fence = sync_fence;
 133
 134                         spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
 135                         if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
 136                                 schedule = MALI_TRUE;
 137                         hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
 138                         spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
 139
 140                         if (schedule)
 141                                 schedule_delayed_work(&delayed_sync_fence_put, 0);
 142                 }
 143         }
 144 #else
 145         sync_fence_put(sync_fence);
 146 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */
 147
 148         if (!is_aborting) {
 149                 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
 150         }
 151 }
 152 #endif /* defined(CONFIG_SYNC) */
 153
 154 static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
 155 {
 156         MALI_DEBUG_ASSERT_POINTER(tracker);
 157         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
 158
 159         return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
 160 }
 161
 162 static void mali_timeline_timer_callback(void *data)
 163 {
 164         struct mali_timeline_system *system;
 165         struct mali_timeline_tracker *tracker;
 166         struct mali_timeline *timeline;
 167         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 168         u32 tid = _mali_osk_get_tid();
 169
 170         timeline = (struct mali_timeline *) data;
 171         MALI_DEBUG_ASSERT_POINTER(timeline);
 172
 173         system = timeline->system;
 174         MALI_DEBUG_ASSERT_POINTER(system);
 175
 176         mali_spinlock_reentrant_wait(system->spinlock, tid);
 177
 178         if (!system->timer_enabled) {
 179                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 180                 return;
 181         }
 182
 183         tracker = timeline->tracker_tail;
 184         timeline->timer_active = MALI_FALSE;
 185
 186         if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
 187                 /* This is likely the delayed work that has been schedule out before cancelled. */
 188                 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
 189                         mali_spinlock_reentrant_signal(system->spinlock, tid);
 190                         return;
 191                 }
 192
 193                 schedule_mask = mali_timeline_tracker_time_out(tracker);
 194                 tracker->timer_active = MALI_FALSE;
 195         } else {
 196                 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
 197         }
 198
 199         mali_spinlock_reentrant_signal(system->spinlock, tid);
 200
 201         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
 202 }
 203
 204 void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
 205 {
 206         u32 i;
 207         u32 tid = _mali_osk_get_tid();
 208
 209         MALI_DEBUG_ASSERT_POINTER(system);
 210
 211         mali_spinlock_reentrant_wait(system->spinlock, tid);
 212         system->timer_enabled = MALI_FALSE;
 213         mali_spinlock_reentrant_signal(system->spinlock, tid);
 214
 215         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 216                 struct mali_timeline *timeline = system->timelines[i];
 217
 218                 MALI_DEBUG_ASSERT_POINTER(timeline);
 219
 220                 if (NULL != timeline->delayed_work) {
 221                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 222                         timeline->timer_active = MALI_FALSE;
 223                 }
 224         }
 225 }
 226
 227 static void mali_timeline_destroy(struct mali_timeline *timeline)
 228 {
 229         MALI_DEBUG_ASSERT_POINTER(timeline);
 230         if (NULL != timeline) {
 231                 /* Assert that the timeline object has been properly cleaned up before destroying it. */
 232                 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 233                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 234                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 235                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 236                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 237                 MALI_DEBUG_ASSERT(NULL != timeline->system);
 238                 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
 239
 240                 if (NULL != timeline->delayed_work) {
 241                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 242                         _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
 243                 }
 244
 245 #if defined(CONFIG_SYNC)
 246                 if (NULL != timeline->sync_tl) {
 247                         sync_timeline_destroy(timeline->sync_tl);
 248                 }
 249 #endif /* defined(CONFIG_SYNC) */
 250
 251 #ifndef CONFIG_SYNC
 252                 _mali_osk_free(timeline);
 253 #endif
 254         }
 255 }
 256
 257 static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
 258 {
 259         struct mali_timeline *timeline;
 260
 261         MALI_DEBUG_ASSERT_POINTER(system);
 262         MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
 263
 264         timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
 265         if (NULL == timeline) {
 266                 return NULL;
 267         }
 268
 269         /* Initially the timeline is empty. */
 270 #if defined(MALI_TIMELINE_DEBUG_START_POINT)
 271         /* Start the timeline a bit before wrapping when debugging. */
 272         timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
 273 #else
 274         timeline->point_next = 1;
 275 #endif
 276         timeline->point_oldest = timeline->point_next;
 277
 278         /* The tracker and waiter lists will initially be empty. */
 279
 280         timeline->system = system;
 281         timeline->id = id;
 282
 283         timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
 284         if (NULL == timeline->delayed_work) {
 285                 mali_timeline_destroy(timeline);
 286                 return NULL;
 287         }
 288
 289         timeline->timer_active = MALI_FALSE;
 290
 291 #if defined(CONFIG_SYNC)
 292         {
 293                 char timeline_name[32];
 294
 295                 switch (id) {
 296                 case MALI_TIMELINE_GP:
 297                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
 298                         break;
 299                 case MALI_TIMELINE_PP:
 300                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
 301                         break;
 302                 case MALI_TIMELINE_SOFT:
 303                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
 304                         break;
 305                 default:
 306                         MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
 307                         mali_timeline_destroy(timeline);
 308                         return NULL;
 309                 }
 310
 311                 timeline->destroyed = MALI_FALSE;
 312
 313                 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
 314                 if (NULL == timeline->sync_tl) {
 315                         mali_timeline_destroy(timeline);
 316                         return NULL;
 317                 }
 318
 319                 timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
 320                 if (NULL == timeline->spinlock) {
 321                         mali_timeline_destroy(timeline);
 322                         return NULL;
 323                 }
 324         }
 325 #endif /* defined(CONFIG_SYNC) */
 326
 327         return timeline;
 328 }
 329
 330 static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
 331 {
 332         MALI_DEBUG_ASSERT_POINTER(timeline);
 333         MALI_DEBUG_ASSERT_POINTER(tracker);
 334
 335         if (mali_timeline_is_full(timeline)) {
 336                 /* Don't add tracker if timeline is full. */
 337                 tracker->point = MALI_TIMELINE_NO_POINT;
 338                 return;
 339         }
 340
 341         tracker->timeline = timeline;
 342         tracker->point    = timeline->point_next;
 343
 344         /* Find next available point. */
 345         timeline->point_next++;
 346         if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
 347                 timeline->point_next++;
 348         }
 349
 350         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 351
 352         if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
 353                 _mali_osk_atomic_inc(&gp_tracker_count);
 354         } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
 355                 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
 356                         _mali_osk_atomic_inc(&virt_pp_tracker_count);
 357                 } else {
 358                         _mali_osk_atomic_inc(&phy_pp_tracker_count);
 359                 }
 360         }
 361
 362         /* Add tracker as new head on timeline's tracker list. */
 363         if (NULL == timeline->tracker_head) {
 364                 /* Tracker list is empty. */
 365                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 366
 367                 timeline->tracker_tail = tracker;
 368
 369                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 370                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
 371         } else {
 372                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 373
 374                 tracker->timeline_prev = timeline->tracker_head;
 375                 timeline->tracker_head->timeline_next = tracker;
 376
 377                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 378         }
 379         timeline->tracker_head = tracker;
 380
 381         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 382         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
 383 }
 384
 385 /* Inserting the waiter object into the given timeline */
 386 static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
 387 {
 388         struct mali_timeline_waiter *waiter_prev;
 389         struct mali_timeline_waiter *waiter_next;
 390
 391         /* Waiter time must be between timeline head and tail, and there must
 392          * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
 393         MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
 394         MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
 395
 396         /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
 397         waiter_prev = timeline->waiter_head; /* Insert new after  waiter_prev */
 398         waiter_next = NULL;                  /* Insert new before waiter_next */
 399
 400         /* Iterating backwards from head (newest) to tail (oldest) until we
 401          * find the correct spot to insert the new waiter */
 402         while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
 403                 waiter_next = waiter_prev;
 404                 waiter_prev = waiter_prev->timeline_prev;
 405         }
 406
 407         if (NULL == waiter_prev && NULL == waiter_next) {
 408                 /* list is empty */
 409                 timeline->waiter_head = waiter_new;
 410                 timeline->waiter_tail = waiter_new;
 411         } else if (NULL == waiter_next) {
 412                 /* insert at head */
 413                 waiter_new->timeline_prev = timeline->waiter_head;
 414                 timeline->waiter_head->timeline_next = waiter_new;
 415                 timeline->waiter_head = waiter_new;
 416         } else if (NULL == waiter_prev) {
 417                 /* insert at tail */
 418                 waiter_new->timeline_next = timeline->waiter_tail;
 419                 timeline->waiter_tail->timeline_prev = waiter_new;
 420                 timeline->waiter_tail = waiter_new;
 421         } else {
 422                 /* insert between */
 423                 waiter_new->timeline_next = waiter_next;
 424                 waiter_new->timeline_prev = waiter_prev;
 425                 waiter_next->timeline_prev = waiter_new;
 426                 waiter_prev->timeline_next = waiter_new;
 427         }
 428 }
 429
 430 static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
 431 {
 432         struct mali_timeline_system *system;
 433         struct mali_timeline_tracker *oldest_tracker;
 434
 435         MALI_DEBUG_ASSERT_POINTER(timeline);
 436         MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
 437
 438         system = timeline->system;
 439         MALI_DEBUG_ASSERT_POINTER(system);
 440
 441         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 442
 443         /* Timer is disabled, early out. */
 444         if (!system->timer_enabled) return;
 445
 446         oldest_tracker = timeline->tracker_tail;
 447         if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
 448                 if (MALI_FALSE == oldest_tracker->timer_active) {
 449                         if (MALI_TRUE == timeline->timer_active) {
 450                                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 451                         }
 452                         _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
 453                         oldest_tracker->timer_active = MALI_TRUE;
 454                         timeline->timer_active = MALI_TRUE;
 455                 }
 456         } else if (MALI_TRUE == timeline->timer_active) {
 457                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 458                 timeline->timer_active = MALI_FALSE;
 459         }
 460 }
 461
 462 static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
 463 {
 464         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 465
 466         MALI_DEBUG_ASSERT_POINTER(timeline);
 467
 468         MALI_DEBUG_CODE({
 469                 struct mali_timeline_system *system = timeline->system;
 470                 MALI_DEBUG_ASSERT_POINTER(system);
 471
 472                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 473         });
 474
 475         if (NULL != timeline->tracker_tail) {
 476                 /* Set oldest point to oldest tracker's point */
 477                 timeline->point_oldest = timeline->tracker_tail->point;
 478         } else {
 479                 /* No trackers, mark point list as empty */
 480                 timeline->point_oldest = timeline->point_next;
 481         }
 482
 483         /* Release all waiters no longer on the timeline's point list.
 484          * Releasing a waiter can trigger this function to be called again, so
 485          * we do not store any pointers on stack. */
 486         while (NULL != timeline->waiter_tail) {
 487                 u32 waiter_time_relative;
 488                 u32 time_head_relative;
 489                 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
 490
 491                 time_head_relative = timeline->point_next - timeline->point_oldest;
 492                 waiter_time_relative = waiter->point - timeline->point_oldest;
 493
 494                 if (waiter_time_relative < time_head_relative) {
 495                         /* This and all following waiters are on the point list, so we are done. */
 496                         break;
 497                 }
 498
 499                 /* Remove waiter from timeline's waiter list. */
 500                 if (NULL != waiter->timeline_next) {
 501                         waiter->timeline_next->timeline_prev = NULL;
 502                 } else {
 503                         /* This was the last waiter */
 504                         timeline->waiter_head = NULL;
 505                 }
 506                 timeline->waiter_tail = waiter->timeline_next;
 507
 508                 /* Release waiter.  This could activate a tracker, if this was
 509                  * the last waiter for the tracker. */
 510                 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
 511         }
 512
 513         return schedule_mask;
 514 }
 515
 516 void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
 517                                 mali_timeline_tracker_type type,
 518                                 struct mali_timeline_fence *fence,
 519                                 void *job)
 520 {
 521         MALI_DEBUG_ASSERT_POINTER(tracker);
 522         MALI_DEBUG_ASSERT_POINTER(job);
 523
 524         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
 525
 526         /* Zero out all tracker members. */
 527         _mali_osk_memset(tracker, 0, sizeof(*tracker));
 528
 529         tracker->type = type;
 530         tracker->job = job;
 531         tracker->trigger_ref_count = 1;  /* Prevents any callback from trigging while adding it */
 532         tracker->os_tick_create = _mali_osk_time_tickcount();
 533         MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
 534
 535         tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
 536
 537         /* Copy fence. */
 538         if (NULL != fence) {
 539                 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
 540         }
 541 }
 542
 543 mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
 544 {
 545         struct mali_timeline *timeline;
 546         struct mali_timeline_system *system;
 547         struct mali_timeline_tracker *tracker_next, *tracker_prev;
 548         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 549         u32 tid = _mali_osk_get_tid();
 550
 551         /* Upon entry a group lock will be held, but not a scheduler lock. */
 552         MALI_DEBUG_ASSERT_POINTER(tracker);
 553         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 554
 555         /* Tracker should have been triggered */
 556         MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
 557
 558         /* All waiters should have been released at this point */
 559         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
 560         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
 561
 562         MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
 563
 564         timeline = tracker->timeline;
 565         if (NULL == timeline) {
 566                 /* Tracker was not on a timeline, there is nothing to release. */
 567                 return MALI_SCHEDULER_MASK_EMPTY;
 568         }
 569
 570         system = timeline->system;
 571         MALI_DEBUG_ASSERT_POINTER(system);
 572
 573         mali_spinlock_reentrant_wait(system->spinlock, tid);
 574
 575         /* Tracker should still be on timeline */
 576         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 577         MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
 578
 579         /* Tracker is no longer valid. */
 580         MALI_DEBUG_CODE(tracker->magic = 0);
 581
 582         tracker_next = tracker->timeline_next;
 583         tracker_prev = tracker->timeline_prev;
 584         tracker->timeline_next = NULL;
 585         tracker->timeline_prev = NULL;
 586
 587         /* Removing tracker from timeline's tracker list */
 588         if (NULL == tracker_next) {
 589                 /* This tracker was the head */
 590                 timeline->tracker_head = tracker_prev;
 591         } else {
 592                 tracker_next->timeline_prev = tracker_prev;
 593         }
 594
 595         if (NULL == tracker_prev) {
 596                 /* This tracker was the tail */
 597                 timeline->tracker_tail = tracker_next;
 598                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 599                 /* Update the timeline's oldest time and release any waiters */
 600                 schedule_mask |= mali_timeline_update_oldest_point(timeline);
 601                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 602         } else {
 603                 tracker_prev->timeline_next = tracker_next;
 604         }
 605
 606         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 607
 608         /* Update delayed work only when it is the soft job timeline */
 609         if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
 610                 mali_timeline_update_delayed_work(tracker->timeline);
 611         }
 612
 613         mali_spinlock_reentrant_signal(system->spinlock, tid);
 614
 615         return schedule_mask;
 616 }
 617
 618 void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
 619                 struct mali_timeline_waiter *tail,
 620                 struct mali_timeline_waiter *head)
 621 {
 622         MALI_DEBUG_ASSERT_POINTER(system);
 623         MALI_DEBUG_ASSERT_POINTER(head);
 624         MALI_DEBUG_ASSERT_POINTER(tail);
 625         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 626
 627         head->tracker_next = system->waiter_empty_list;
 628         system->waiter_empty_list = tail;
 629 }
 630
 631 static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
 632 {
 633         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 634         struct mali_timeline_system *system;
 635         struct mali_timeline *timeline;
 636         u32 tid = _mali_osk_get_tid();
 637
 638         MALI_DEBUG_ASSERT_POINTER(tracker);
 639         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 640
 641         system = tracker->system;
 642         MALI_DEBUG_ASSERT_POINTER(system);
 643         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 644
 645         tracker->os_tick_activate = _mali_osk_time_tickcount();
 646
 647         if (NULL != tracker->waiter_head) {
 648                 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
 649                 tracker->waiter_head = NULL;
 650                 tracker->waiter_tail = NULL;
 651         }
 652
 653         switch (tracker->type) {
 654         case MALI_TIMELINE_TRACKER_GP:
 655                 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
 656
 657                 _mali_osk_atomic_dec(&gp_tracker_count);
 658                 break;
 659         case MALI_TIMELINE_TRACKER_PP:
 660                 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
 661                         _mali_osk_atomic_dec(&virt_pp_tracker_count);
 662                 } else {
 663                         _mali_osk_atomic_dec(&phy_pp_tracker_count);
 664                 }
 665                 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
 666                 break;
 667         case MALI_TIMELINE_TRACKER_SOFT:
 668                 timeline = tracker->timeline;
 669                 MALI_DEBUG_ASSERT_POINTER(timeline);
 670
 671                 schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
 672
 673                 /* Start a soft timer to make sure the soft job be released in a limited time */
 674                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 675                 mali_timeline_update_delayed_work(timeline);
 676                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 677                 break;
 678         case MALI_TIMELINE_TRACKER_WAIT:
 679                 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
 680                 break;
 681         case MALI_TIMELINE_TRACKER_SYNC:
 682 #if defined(CONFIG_SYNC)
 683                 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
 684 #else
 685                 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
 686 #endif /* defined(CONFIG_SYNC) */
 687                 break;
 688         default:
 689                 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
 690                 break;
 691         }
 692
 693         return schedule_mask;
 694 }
 695
 696 void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
 697 {
 698         u32 tid = _mali_osk_get_tid();
 699
 700         MALI_DEBUG_ASSERT_POINTER(tracker);
 701         MALI_DEBUG_ASSERT_POINTER(system);
 702
 703         mali_spinlock_reentrant_wait(system->spinlock, tid);
 704
 705         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 706         tracker->trigger_ref_count++;
 707
 708         mali_spinlock_reentrant_signal(system->spinlock, tid);
 709 }
 710
 711 mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
 712 {
 713         u32 tid = _mali_osk_get_tid();
 714         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 715
 716         MALI_DEBUG_ASSERT_POINTER(tracker);
 717         MALI_DEBUG_ASSERT_POINTER(system);
 718
 719         mali_spinlock_reentrant_wait(system->spinlock, tid);
 720
 721         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 722         tracker->trigger_ref_count--;
 723
 724         tracker->activation_error |= activation_error;
 725
 726         if (0 == tracker->trigger_ref_count) {
 727                 schedule_mask |= mali_timeline_tracker_activate(tracker);
 728                 tracker = NULL;
 729         }
 730
 731         mali_spinlock_reentrant_signal(system->spinlock, tid);
 732
 733         return schedule_mask;
 734 }
 735
 736 void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
 737 {
 738         u32 i;
 739
 740         MALI_DEBUG_ASSERT_POINTER(fence);
 741         MALI_DEBUG_ASSERT_POINTER(uk_fence);
 742
 743         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 744                 fence->points[i] = uk_fence->points[i];
 745         }
 746
 747         fence->sync_fd = uk_fence->sync_fd;
 748 }
 749
 750 struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
 751 {
 752         u32 i;
 753         struct mali_timeline_system *system;
 754
 755         MALI_DEBUG_ASSERT_POINTER(session);
 756         MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
 757
 758         system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
 759         if (NULL == system) {
 760                 return NULL;
 761         }
 762
 763         system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
 764         if (NULL == system->spinlock) {
 765                 mali_timeline_system_destroy(system);
 766                 return NULL;
 767         }
 768
 769         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 770                 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
 771                 if (NULL == system->timelines[i]) {
 772                         mali_timeline_system_destroy(system);
 773                         return NULL;
 774                 }
 775         }
 776
 777 #if defined(CONFIG_SYNC)
 778         system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
 779         if (NULL == system->signaled_sync_tl) {
 780                 mali_timeline_system_destroy(system);
 781                 return NULL;
 782         }
 783 #endif /* defined(CONFIG_SYNC) */
 784
 785         system->waiter_empty_list = NULL;
 786         system->session = session;
 787         system->timer_enabled = MALI_TRUE;
 788
 789         system->wait_queue = _mali_osk_wait_queue_init();
 790         if (NULL == system->wait_queue) {
 791                 mali_timeline_system_destroy(system);
 792                 return NULL;
 793         }
 794
 795         return system;
 796 }
 797
 798 #if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC)
 799 /**
 800  * Check if there are any trackers left on timeline.
 801  *
 802  * Used as a wait queue conditional.
 803  *
 804  * @param data Timeline.
 805  * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
 806  */
 807 static mali_bool mali_timeline_has_no_trackers(void *data)
 808 {
 809         struct mali_timeline *timeline = (struct mali_timeline *) data;
 810
 811         MALI_DEBUG_ASSERT_POINTER(timeline);
 812
 813         return mali_timeline_is_empty(timeline);
 814 }
 815 #if defined(CONFIG_SYNC)
 816 /**
 817  * Cancel sync fence waiters waited upon by trackers on all timelines.
 818  *
 819  * Will return after all timelines have no trackers left.
 820  *
 821  * @param system Timeline system.
 822  */
 823 static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
 824 {
 825         u32 i;
 826         u32 tid = _mali_osk_get_tid();
 827         struct mali_timeline_tracker *tracker, *tracker_next;
 828         _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
 829
 830         MALI_DEBUG_ASSERT_POINTER(system);
 831         MALI_DEBUG_ASSERT_POINTER(system->session);
 832         MALI_DEBUG_ASSERT(system->session->is_aborting);
 833
 834         mali_spinlock_reentrant_wait(system->spinlock, tid);
 835
 836         /* Cancel sync fence waiters. */
 837         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 838                 struct mali_timeline *timeline = system->timelines[i];
 839
 840                 MALI_DEBUG_ASSERT_POINTER(timeline);
 841
 842                 tracker_next = timeline->tracker_tail;
 843                 while (NULL != tracker_next) {
 844                         tracker = tracker_next;
 845                         tracker_next = tracker->timeline_next;
 846
 847                         if (NULL == tracker->sync_fence) continue;
 848
 849                         MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
 850
 851                         /* Cancel sync fence waiter. */
 852                         if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
 853                                 /* Callback was not called, move tracker to local list. */
 854                                 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
 855                         }
 856                 }
 857         }
 858
 859         mali_spinlock_reentrant_signal(system->spinlock, tid);
 860
 861         /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
 862         _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
 863                 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
 864         }
 865
 866         /* Sleep until all sync fence callbacks are done and all timelines are empty. */
 867         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 868                 struct mali_timeline *timeline = system->timelines[i];
 869
 870                 MALI_DEBUG_ASSERT_POINTER(timeline);
 871
 872                 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
 873         }
 874 }
 875
 876 #endif /* defined(CONFIG_SYNC) */
 877
 878 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
 879 static void mali_timeline_cancel_dma_fence_waiters(struct mali_timeline_system *system)
 880 {
 881         u32 i, j;
 882         u32 tid = _mali_osk_get_tid();
 883         struct mali_pp_job *pp_job = NULL;
 884         struct mali_pp_job *next_pp_job = NULL;
 885         struct mali_timeline *timeline = NULL;
 886         struct mali_timeline_tracker *tracker, *tracker_next;
 887         _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_job_list);
 888
 889         MALI_DEBUG_ASSERT_POINTER(system);
 890         MALI_DEBUG_ASSERT_POINTER(system->session);
 891         MALI_DEBUG_ASSERT(system->session->is_aborting);
 892
 893         mali_spinlock_reentrant_wait(system->spinlock, tid);
 894
 895         /* Cancel dma fence waiters. */
 896         timeline = system->timelines[MALI_TIMELINE_PP];
 897         MALI_DEBUG_ASSERT_POINTER(timeline);
 898
 899         tracker_next = timeline->tracker_tail;
 900         while (NULL != tracker_next) {
 901                 mali_bool fence_is_signaled = MALI_TRUE;
 902                 tracker = tracker_next;
 903                 tracker_next = tracker->timeline_next;
 904
 905                 if (NULL == tracker->waiter_dma_fence) continue;
 906                 pp_job = (struct mali_pp_job *)tracker->job;
 907                 MALI_DEBUG_ASSERT_POINTER(pp_job);
 908                 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling dma fence waiter for tracker 0x%08X.\n", tracker));
 909
 910                 for (j = 0; j < pp_job->dma_fence_context.num_dma_fence_waiter; j++) {
 911                         if (pp_job->dma_fence_context.mali_dma_fence_waiters[j]) {
 912                                 /* Cancel a previously callback from the fence.
 913                                 * This function returns true if the callback is successfully removed,
 914                                 * or false if the fence has already been signaled.
 915                                 */
 916                                 bool ret = fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence,
 917                                                                  &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base);
 918                                 if (ret) {
 919                                         fence_is_signaled = MALI_FALSE;
 920                                 }
 921                         }
 922                 }
 923
 924                 /* Callbacks were not called, move pp job to local list. */
 925                 if (MALI_FALSE == fence_is_signaled)
 926                         _mali_osk_list_add(&pp_job->list, &pp_job_list);
 927         }
 928
 929         mali_spinlock_reentrant_signal(system->spinlock, tid);
 930
 931         /* Manually call dma fence callback in order to release waiter and trigger activation of tracker. */
 932         _MALI_OSK_LIST_FOREACHENTRY(pp_job, next_pp_job, &pp_job_list, struct mali_pp_job, list) {
 933                 mali_timeline_dma_fence_callback((void *)pp_job);
 934         }
 935
 936         /* Sleep until all dma fence callbacks are done and all timelines are empty. */
 937         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 938                 struct mali_timeline *timeline = system->timelines[i];
 939                 MALI_DEBUG_ASSERT_POINTER(timeline);
 940                 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
 941         }
 942 }
 943 #endif
 944 #endif
 945 void mali_timeline_system_abort(struct mali_timeline_system *system)
 946 {
 947         MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
 948
 949         MALI_DEBUG_ASSERT_POINTER(system);
 950         MALI_DEBUG_ASSERT_POINTER(system->session);
 951         MALI_DEBUG_ASSERT(system->session->is_aborting);
 952
 953         MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
 954
 955 #if defined(CONFIG_SYNC)
 956         mali_timeline_cancel_sync_fence_waiters(system);
 957 #endif /* defined(CONFIG_SYNC) */
 958
 959 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
 960         mali_timeline_cancel_dma_fence_waiters(system);
 961 #endif
 962
 963         /* Should not be any waiters or trackers left at this point. */
 964         MALI_DEBUG_CODE({
 965                 u32 i;
 966                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 967                 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
 968                 {
 969                         struct mali_timeline *timeline = system->timelines[i];
 970                         MALI_DEBUG_ASSERT_POINTER(timeline);
 971                         MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 972                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 973                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 974                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 975                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 976                 }
 977                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 978         });
 979 }
 980
 981 void mali_timeline_system_destroy(struct mali_timeline_system *system)
 982 {
 983         u32 i;
 984         struct mali_timeline_waiter *waiter, *next;
 985 #if defined(CONFIG_SYNC)
 986         u32 tid = _mali_osk_get_tid();
 987 #endif
 988
 989         MALI_DEBUG_ASSERT_POINTER(system);
 990         MALI_DEBUG_ASSERT_POINTER(system->session);
 991
 992         MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
 993
 994         if (NULL != system) {
 995
 996                 /* There should be no waiters left on this queue. */
 997                 if (NULL != system->wait_queue) {
 998                         _mali_osk_wait_queue_term(system->wait_queue);
 999                         system->wait_queue = NULL;
1000                 }
1001
1002                 /* Free all waiters in empty list */
1003                 waiter = system->waiter_empty_list;
1004                 while (NULL != waiter) {
1005                         next = waiter->tracker_next;
1006                         _mali_osk_free(waiter);
1007                         waiter = next;
1008                 }
1009
1010 #if defined(CONFIG_SYNC)
1011                 if (NULL != system->signaled_sync_tl) {
1012                         sync_timeline_destroy(system->signaled_sync_tl);
1013                 }
1014
1015                 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1016                         if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) {
1017                                 mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid);
1018                                 system->timelines[i]->destroyed = MALI_TRUE;
1019                                 mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid);
1020                         }
1021                 }
1022 #endif /* defined(CONFIG_SYNC) */
1023
1024                 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1025                         if (NULL != system->timelines[i]) {
1026                                 mali_timeline_destroy(system->timelines[i]);
1027                         }
1028                 }
1029
1030                 if (NULL != system->spinlock) {
1031                         mali_spinlock_reentrant_term(system->spinlock);
1032                 }
1033
1034                 _mali_osk_free(system);
1035         }
1036 }
1037
1038 /**
1039  * Find how many waiters are needed for a given fence.
1040  *
1041  * @param fence The fence to check.
1042  * @return Number of waiters needed for fence.
1043  */
1044 static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
1045 {
1046         u32 i, num_waiters = 0;
1047
1048         MALI_DEBUG_ASSERT_POINTER(fence);
1049
1050         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1051                 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
1052                         ++num_waiters;
1053                 }
1054         }
1055
1056 #if defined(CONFIG_SYNC)
1057         if (-1 != fence->sync_fd) ++num_waiters;
1058 #endif /* defined(CONFIG_SYNC) */
1059
1060         return num_waiters;
1061 }
1062
1063 static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
1064 {
1065         struct mali_timeline_waiter *waiter;
1066
1067         MALI_DEBUG_ASSERT_POINTER(system);
1068         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1069
1070         waiter = system->waiter_empty_list;
1071         if (NULL != waiter) {
1072                 /* Remove waiter from empty list and zero it */
1073                 system->waiter_empty_list = waiter->tracker_next;
1074                 _mali_osk_memset(waiter, 0, sizeof(*waiter));
1075         }
1076
1077         /* Return NULL if list was empty. */
1078         return waiter;
1079 }
1080
1081 static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
1082                 struct mali_timeline_waiter **tail,
1083                 struct mali_timeline_waiter **head,
1084                 int max_num_waiters)
1085 {
1086         u32 i, tid = _mali_osk_get_tid();
1087         mali_bool do_alloc;
1088         struct mali_timeline_waiter *waiter;
1089
1090         MALI_DEBUG_ASSERT_POINTER(system);
1091         MALI_DEBUG_ASSERT_POINTER(tail);
1092         MALI_DEBUG_ASSERT_POINTER(head);
1093
1094         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1095
1096         *head = *tail = NULL;
1097         do_alloc = MALI_FALSE;
1098         i = 0;
1099         while (i < max_num_waiters) {
1100                 if (MALI_FALSE == do_alloc) {
1101                         waiter = mali_timeline_system_get_zeroed_waiter(system);
1102                         if (NULL == waiter) {
1103                                 do_alloc = MALI_TRUE;
1104                                 mali_spinlock_reentrant_signal(system->spinlock, tid);
1105                                 continue;
1106                         }
1107                 } else {
1108                         waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1109                         if (NULL == waiter) break;
1110                 }
1111                 ++i;
1112                 if (NULL == *tail) {
1113                         *tail = waiter;
1114                         *head = waiter;
1115                 } else {
1116                         (*head)->tracker_next = waiter;
1117                         *head = waiter;
1118                 }
1119         }
1120         if (MALI_TRUE == do_alloc) {
1121                 mali_spinlock_reentrant_wait(system->spinlock, tid);
1122         }
1123 }
1124
1125 /**
1126  * Create waiters for the given tracker. The tracker is activated when all waiters are release.
1127  *
1128  * @note Tracker can potentially be activated before this function returns.
1129  *
1130  * @param system Timeline system.
1131  * @param tracker Tracker we will create waiters for.
1132  * @param waiter_tail List of pre-allocated waiters.
1133  * @param waiter_head List of pre-allocated waiters.
1134  */
1135 static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
1136                 struct mali_timeline_tracker *tracker,
1137                 struct mali_timeline_waiter *waiter_tail,
1138                 struct mali_timeline_waiter *waiter_head)
1139 {
1140         int i;
1141         u32 tid = _mali_osk_get_tid();
1142         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1143 #if defined(CONFIG_SYNC)
1144         struct sync_fence *sync_fence = NULL;
1145 #endif /* defined(CONFIG_SYNC) */
1146
1147         MALI_DEBUG_ASSERT_POINTER(system);
1148         MALI_DEBUG_ASSERT_POINTER(tracker);
1149
1150         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1151
1152         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1153         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1154         MALI_DEBUG_ASSERT(NULL != tracker->job);
1155
1156         /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
1157          * into the timelines sorted list of waiters */
1158         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1159                 mali_timeline_point point;
1160                 struct mali_timeline *timeline;
1161                 struct mali_timeline_waiter *waiter;
1162
1163                 /* Get point on current timeline from tracker's fence. */
1164                 point = tracker->fence.points[i];
1165
1166                 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1167                         /* Fence contains no point on this timeline so we don't need a waiter. */
1168                         continue;
1169                 }
1170
1171                 timeline = system->timelines[i];
1172                 MALI_DEBUG_ASSERT_POINTER(timeline);
1173
1174                 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1175                         MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
1176                                           point, timeline->point_oldest, timeline->point_next));
1177                         continue;
1178                 }
1179
1180                 if (likely(mali_timeline_is_point_released(timeline, point))) {
1181                         /* Tracker representing the point has been released so we don't need a
1182                          * waiter. */
1183                         continue;
1184                 }
1185
1186                 /* The point is on timeline. */
1187                 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1188
1189                 /* Get a new zeroed waiter object. */
1190                 if (likely(NULL != waiter_tail)) {
1191                         waiter = waiter_tail;
1192                         waiter_tail = waiter_tail->tracker_next;
1193                 } else {
1194                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1195                         continue;
1196                 }
1197
1198                 /* Yanking the trigger ref count of the tracker. */
1199                 tracker->trigger_ref_count++;
1200
1201                 waiter->point   = point;
1202                 waiter->tracker = tracker;
1203
1204                 /* Insert waiter on tracker's singly-linked waiter list. */
1205                 if (NULL == tracker->waiter_head) {
1206                         /* list is empty */
1207                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1208                         tracker->waiter_tail = waiter;
1209                 } else {
1210                         tracker->waiter_head->tracker_next = waiter;
1211                 }
1212                 tracker->waiter_head = waiter;
1213
1214                 /* Add waiter to timeline. */
1215                 mali_timeline_insert_waiter(timeline, waiter);
1216         }
1217 #if defined(CONFIG_SYNC)
1218         if (-1 != tracker->fence.sync_fd) {
1219                 int ret;
1220                 struct mali_timeline_waiter *waiter;
1221
1222                 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1223                 if (unlikely(NULL == sync_fence)) {
1224                         MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1225                         goto exit;
1226                 }
1227
1228                 /* Check if we have a zeroed waiter object available. */
1229                 if (unlikely(NULL == waiter_tail)) {
1230                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1231                         goto exit;
1232                 }
1233
1234                 /* Start asynchronous wait that will release waiter when the fence is signaled. */
1235                 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1236                 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1237                 if (1 == ret) {
1238                         /* Fence already signaled, no waiter needed. */
1239                         tracker->fence.sync_fd = -1;
1240                         goto exit;
1241                 } else if (0 != ret) {
1242                         MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1243                         tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1244                         goto exit;
1245                 }
1246
1247                 /* Grab new zeroed waiter object. */
1248                 waiter = waiter_tail;
1249                 waiter_tail = waiter_tail->tracker_next;
1250
1251                 /* Increase the trigger ref count of the tracker. */
1252                 tracker->trigger_ref_count++;
1253
1254                 waiter->point   = MALI_TIMELINE_NO_POINT;
1255                 waiter->tracker = tracker;
1256
1257                 /* Insert waiter on tracker's singly-linked waiter list. */
1258                 if (NULL == tracker->waiter_head) {
1259                         /* list is empty */
1260                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1261                         tracker->waiter_tail = waiter;
1262                 } else {
1263                         tracker->waiter_head->tracker_next = waiter;
1264                 }
1265                 tracker->waiter_head = waiter;
1266
1267                 /* Also store waiter in separate field for easy access by sync callback. */
1268                 tracker->waiter_sync = waiter;
1269
1270                 /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
1271                 tracker->sync_fence = sync_fence;
1272
1273                 sync_fence = NULL;
1274         }
1275 #endif /* defined(CONFIG_SYNC)*/
1276 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1277         if ((NULL != tracker->timeline) && (MALI_TIMELINE_PP == tracker->timeline->id)) {
1278
1279                 struct mali_pp_job *job = (struct mali_pp_job *)tracker->job;
1280
1281                 if (0 < job->dma_fence_context.num_dma_fence_waiter) {
1282                         struct mali_timeline_waiter *waiter;
1283                         /* Check if we have a zeroed waiter object available. */
1284                         if (unlikely(NULL == waiter_tail)) {
1285                                 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1286                                 goto exit;
1287                         }
1288
1289                         /* Grab new zeroed waiter object. */
1290                         waiter = waiter_tail;
1291                         waiter_tail = waiter_tail->tracker_next;
1292
1293                         /* Increase the trigger ref count of the tracker. */
1294                         tracker->trigger_ref_count++;
1295
1296                         waiter->point   = MALI_TIMELINE_NO_POINT;
1297                         waiter->tracker = tracker;
1298
1299                         /* Insert waiter on tracker's singly-linked waiter list. */
1300                         if (NULL == tracker->waiter_head) {
1301                                 /* list is empty */
1302                                 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1303                                 tracker->waiter_tail = waiter;
1304                         } else {
1305                                 tracker->waiter_head->tracker_next = waiter;
1306                         }
1307                         tracker->waiter_head = waiter;
1308
1309                         /* Also store waiter in separate field for easy access by sync callback. */
1310                         tracker->waiter_dma_fence = waiter;
1311                 }
1312         }
1313 #endif /* defined(CONFIG_MALI_DMA_BUF_FENCE)*/
1314
1315 #if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC)
1316 exit:
1317 #endif /* defined(CONFIG_MALI_DMA_BUF_FENCE) || defined(CONFIG_SYNC) */
1318
1319         if (NULL != waiter_tail) {
1320                 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1321         }
1322
1323         /* Release the initial trigger ref count. */
1324         tracker->trigger_ref_count--;
1325
1326         /* If there were no waiters added to this tracker we activate immediately. */
1327         if (0 == tracker->trigger_ref_count) {
1328                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1329         }
1330
1331         mali_spinlock_reentrant_signal(system->spinlock, tid);
1332
1333 #if defined(CONFIG_SYNC)
1334         if (NULL != sync_fence) {
1335                 sync_fence_put(sync_fence);
1336         }
1337 #endif /* defined(CONFIG_SYNC) */
1338
1339         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1340 }
1341
1342 mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1343                 struct mali_timeline_tracker *tracker,
1344                 enum mali_timeline_id timeline_id)
1345 {
1346         int num_waiters = 0;
1347         struct mali_timeline_waiter *waiter_tail, *waiter_head;
1348         u32 tid = _mali_osk_get_tid();
1349
1350         mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1351
1352         MALI_DEBUG_ASSERT_POINTER(system);
1353         MALI_DEBUG_ASSERT_POINTER(system->session);
1354         MALI_DEBUG_ASSERT_POINTER(tracker);
1355
1356         MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1357         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1358         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1359
1360         MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1361
1362         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1363         tracker->system = system;
1364
1365         mali_spinlock_reentrant_wait(system->spinlock, tid);
1366
1367         num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1368
1369 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1370         if (MALI_TIMELINE_PP == timeline_id) {
1371                 struct mali_pp_job *job = (struct mali_pp_job *)tracker->job;
1372                 if (0 < job->dma_fence_context.num_dma_fence_waiter)
1373                         num_waiters++;
1374         }
1375 #endif
1376
1377         /* Allocate waiters. */
1378         mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1379         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1380
1381         /* Add tracker to timeline.  This will allocate a point for the tracker on the timeline. If
1382          * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
1383          * point will be MALI_TIMELINE_NO_POINT.
1384          *
1385          * NOTE: the tracker can fail to be added if the timeline is full.  If this happens, the
1386          * point will be MALI_TIMELINE_NO_POINT. */
1387         MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1388         if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1389                 struct mali_timeline *timeline = system->timelines[timeline_id];
1390                 mali_timeline_insert_tracker(timeline, tracker);
1391                 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1392         }
1393
1394         point = tracker->point;
1395
1396         /* Create waiters for tracker based on supplied fence.  Each waiter will increase the
1397          * trigger ref count. */
1398         mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1399         tracker = NULL;
1400
1401         /* At this point the tracker object might have been freed so we should no longer
1402          * access it. */
1403
1404
1405         /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
1406          * returned. */
1407         return point;
1408 }
1409
1410 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1411                 struct mali_timeline_waiter *waiter)
1412 {
1413         struct mali_timeline_tracker *tracker;
1414         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1415
1416         MALI_DEBUG_ASSERT_POINTER(system);
1417         MALI_DEBUG_ASSERT_POINTER(waiter);
1418
1419         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1420
1421         tracker = waiter->tracker;
1422         MALI_DEBUG_ASSERT_POINTER(tracker);
1423
1424         /* At this point the waiter has been removed from the timeline's waiter list, but it is
1425          * still on the tracker's waiter list.  All of the tracker's waiters will be released when
1426          * the tracker is activated. */
1427
1428         waiter->point   = MALI_TIMELINE_NO_POINT;
1429         waiter->tracker = NULL;
1430
1431         tracker->trigger_ref_count--;
1432         if (0 == tracker->trigger_ref_count) {
1433                 /* This was the last waiter; activate tracker */
1434                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1435                 tracker = NULL;
1436         }
1437
1438         return schedule_mask;
1439 }
1440
1441 mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1442                 enum mali_timeline_id timeline_id)
1443 {
1444         mali_timeline_point point;
1445         struct mali_timeline *timeline;
1446         u32 tid = _mali_osk_get_tid();
1447
1448         MALI_DEBUG_ASSERT_POINTER(system);
1449
1450         if (MALI_TIMELINE_MAX <= timeline_id) {
1451                 return MALI_TIMELINE_NO_POINT;
1452         }
1453
1454         mali_spinlock_reentrant_wait(system->spinlock, tid);
1455
1456         timeline = system->timelines[timeline_id];
1457         MALI_DEBUG_ASSERT_POINTER(timeline);
1458
1459         point = MALI_TIMELINE_NO_POINT;
1460         if (timeline->point_oldest != timeline->point_next) {
1461                 point = timeline->point_next - 1;
1462                 if (MALI_TIMELINE_NO_POINT == point) point--;
1463         }
1464
1465         mali_spinlock_reentrant_signal(system->spinlock, tid);
1466
1467         return point;
1468 }
1469
1470 void mali_timeline_initialize(void)
1471 {
1472         _mali_osk_atomic_init(&gp_tracker_count, 0);
1473         _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1474         _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1475 }
1476
1477 void mali_timeline_terminate(void)
1478 {
1479         _mali_osk_atomic_term(&gp_tracker_count);
1480         _mali_osk_atomic_term(&phy_pp_tracker_count);
1481         _mali_osk_atomic_term(&virt_pp_tracker_count);
1482 }
1483
1484 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1485
1486 static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1487 {
1488         struct mali_timeline *timeline;
1489         struct mali_timeline_system *system;
1490
1491         MALI_DEBUG_ASSERT_POINTER(tracker);
1492
1493         MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1494         timeline = tracker->timeline;
1495
1496         MALI_DEBUG_ASSERT_POINTER(timeline->system);
1497         system = timeline->system;
1498
1499         if (MALI_TIMELINE_MAX > id) {
1500                 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1501                         return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1502                 } else {
1503                         return MALI_FALSE;
1504                 }
1505         } else {
1506                 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1507                 return MALI_FALSE;
1508         }
1509 }
1510
1511 static const char *timeline_id_to_string(enum mali_timeline_id id)
1512 {
1513         switch (id) {
1514         case MALI_TIMELINE_GP:
1515                 return "GP";
1516         case MALI_TIMELINE_PP:
1517                 return "PP";
1518         case MALI_TIMELINE_SOFT:
1519                 return "SOFT";
1520         default:
1521                 return "NONE";
1522         }
1523 }
1524
1525 static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1526 {
1527         switch (type) {
1528         case MALI_TIMELINE_TRACKER_GP:
1529                 return "GP";
1530         case MALI_TIMELINE_TRACKER_PP:
1531                 return "PP";
1532         case MALI_TIMELINE_TRACKER_SOFT:
1533                 return "SOFT";
1534         case MALI_TIMELINE_TRACKER_WAIT:
1535                 return "WAIT";
1536         case MALI_TIMELINE_TRACKER_SYNC:
1537                 return "SYNC";
1538         default:
1539                 return "INVALID";
1540         }
1541 }
1542
1543 mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1544 {
1545         struct mali_timeline *timeline = NULL;
1546
1547         MALI_DEBUG_ASSERT_POINTER(tracker);
1548         timeline = tracker->timeline;
1549
1550         if (0 != tracker->trigger_ref_count) {
1551                 return MALI_TIMELINE_TS_WAITING;
1552         }
1553
1554         if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1555                 return MALI_TIMELINE_TS_ACTIVE;
1556         }
1557
1558         if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1559                 return MALI_TIMELINE_TS_INIT;
1560         }
1561
1562         return MALI_TIMELINE_TS_FINISH;
1563 }
1564
1565 void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
1566 {
1567         const char *tracker_state = "IWAF";
1568         char state_char = 'I';
1569         char tracker_type[32] = {0};
1570
1571         MALI_DEBUG_ASSERT_POINTER(tracker);
1572
1573         state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1574         _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1575
1576 #if defined(CONFIG_SYNC)
1577         if (0 != tracker->trigger_ref_count) {
1578                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)]  job:(0x%08X)\n",
1579                                     tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1580                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1581                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1582                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1583                                     tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job));
1584         } else {
1585                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c  fd:%d  fence:(0x%08X)  job:(0x%08X)\n",
1586                                     tracker_type, tracker->point, state_char,
1587                                     tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job));
1588         }
1589 #else
1590         if (0 != tracker->trigger_ref_count) {
1591                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)]  job:(0x%08X)\n",
1592                                     tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1593                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1594                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1595                                     is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1596                                     (unsigned int)(uintptr_t)(tracker->job));
1597         } else {
1598                 _mali_osk_ctxprintf(print_ctx, "TL:  %s %u %c  job:(0x%08X)\n",
1599                                     tracker_type, tracker->point, state_char,
1600                                     (unsigned int)(uintptr_t)(tracker->job));
1601         }
1602 #endif
1603 }
1604
1605 void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
1606 {
1607         struct mali_timeline_tracker *tracker = NULL;
1608
1609         MALI_DEBUG_ASSERT_POINTER(timeline);
1610
1611         tracker = timeline->tracker_tail;
1612         while (NULL != tracker) {
1613                 mali_timeline_debug_print_tracker(tracker, print_ctx);
1614                 tracker = tracker->timeline_next;
1615         }
1616 }
1617
1618 #if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))
1619 void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker)
1620 {
1621         const char *tracker_state = "IWAF";
1622         char state_char = 'I';
1623         char tracker_type[32] = {0};
1624
1625         MALI_DEBUG_ASSERT_POINTER(tracker);
1626
1627         state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1628         _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1629
1630 #if defined(CONFIG_SYNC)
1631         if (0 != tracker->trigger_ref_count) {
1632                 MALI_PRINT(("TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)]  job:(0x%08X)\n",
1633                             tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1634                             is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1635                             is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1636                             is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1637                             tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1638         } else {
1639                 MALI_PRINT(("TL:  %s %u %c  fd:%d  fence:(0x%08X)  job:(0x%08X)\n",
1640                             tracker_type, tracker->point, state_char,
1641                             tracker->fence.sync_fd, tracker->sync_fence, tracker->job));
1642         }
1643 #else
1644         if (0 != tracker->trigger_ref_count) {
1645                 MALI_PRINT(("TL:  %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)]  job:(0x%08X)\n",
1646                             tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1647                             is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1648                             is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1649                             is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1650                             tracker->job));
1651         } else {
1652                 MALI_PRINT(("TL:  %s %u %c  job:(0x%08X)\n",
1653                             tracker_type, tracker->point, state_char,
1654                             tracker->job));
1655         }
1656 #endif
1657 }
1658
1659 void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline)
1660 {
1661         struct mali_timeline_tracker *tracker = NULL;
1662
1663         MALI_DEBUG_ASSERT_POINTER(timeline);
1664
1665         tracker = timeline->tracker_tail;
1666         while (NULL != tracker) {
1667                 mali_timeline_debug_direct_print_tracker(tracker);
1668                 tracker = tracker->timeline_next;
1669         }
1670 }
1671
1672 #endif
1673
1674 void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
1675 {
1676         int i;
1677         int num_printed = 0;
1678         u32 tid = _mali_osk_get_tid();
1679
1680         MALI_DEBUG_ASSERT_POINTER(system);
1681
1682         mali_spinlock_reentrant_wait(system->spinlock, tid);
1683
1684         /* Print all timelines */
1685         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1686                 struct mali_timeline *timeline = system->timelines[i];
1687
1688                 MALI_DEBUG_ASSERT_POINTER(timeline);
1689
1690                 if (NULL == timeline->tracker_head) continue;
1691
1692                 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1693                                     timeline_id_to_string((enum mali_timeline_id)i));
1694
1695                 mali_timeline_debug_print_timeline(timeline, print_ctx);
1696                 num_printed++;
1697         }
1698
1699         if (0 == num_printed) {
1700                 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
1701         }
1702
1703         mali_spinlock_reentrant_signal(system->spinlock, tid);
1704 }
1705
1706 #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */
1707
1708 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1709 void mali_timeline_dma_fence_callback(void *pp_job_ptr)
1710 {
1711         struct mali_timeline_system  *system;
1712         struct mali_timeline_waiter  *waiter;
1713         struct mali_timeline_tracker *tracker;
1714         struct mali_pp_job *pp_job = (struct mali_pp_job *)pp_job_ptr;
1715         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1716         u32 tid = _mali_osk_get_tid();
1717         mali_bool is_aborting = MALI_FALSE;
1718
1719         MALI_DEBUG_ASSERT_POINTER(pp_job);
1720
1721         tracker = &pp_job->tracker;
1722         MALI_DEBUG_ASSERT_POINTER(tracker);
1723
1724         system = tracker->system;
1725         MALI_DEBUG_ASSERT_POINTER(system);
1726         MALI_DEBUG_ASSERT_POINTER(system->session);
1727
1728         mali_spinlock_reentrant_wait(system->spinlock, tid);
1729
1730         waiter = tracker->waiter_dma_fence;
1731         MALI_DEBUG_ASSERT_POINTER(waiter);
1732
1733         schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
1734
1735         is_aborting = system->session->is_aborting;
1736
1737         /* If aborting, wake up sleepers that are waiting for dma fence callbacks to complete. */
1738         if (is_aborting) {
1739                 _mali_osk_wait_queue_wake_up(system->wait_queue);
1740         }
1741
1742         mali_spinlock_reentrant_signal(system->spinlock, tid);
1743
1744         if (!is_aborting) {
1745                 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
1746         }
1747 }
1748 #endif