drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c

   1 /*
   2  * Copyright (C) 2012-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10 #include <linux/version.h>
  11 #include "mali_osk.h"
  12 #include "mali_kernel_common.h"
  13
  14 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
  15 #include "mali_dma_fence.h"
  16 #include <linux/atomic.h>
  17 #include <linux/workqueue.h>
  18 #endif
  19
  20 static DEFINE_SPINLOCK(mali_dma_fence_lock);
  21
  22 static bool mali_dma_fence_enable_signaling(struct fence *fence)
  23 {
  24         MALI_IGNORE(fence);
  25         return true;
  26 }
  27
  28 static const char *mali_dma_fence_get_driver_name(struct fence *fence)
  29 {
  30         MALI_IGNORE(fence);
  31         return "mali";
  32 }
  33
  34 static const char *mali_dma_fence_get_timeline_name(struct fence *fence)
  35 {
  36         MALI_IGNORE(fence);
  37         return "mali_dma_fence";
  38 }
  39
  40 static const struct fence_ops mali_dma_fence_ops = {
  41         .get_driver_name = mali_dma_fence_get_driver_name,
  42         .get_timeline_name = mali_dma_fence_get_timeline_name,
  43         .enable_signaling = mali_dma_fence_enable_signaling,
  44         .signaled = NULL,
  45         .wait = fence_default_wait,
  46         .release = NULL
  47 };
  48
  49 static void mali_dma_fence_context_cleanup(struct mali_dma_fence_context *dma_fence_context)
  50 {
  51         u32 i;
  52
  53         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
  54
  55         for (i = 0; i < dma_fence_context->num_dma_fence_waiter; i++) {
  56                 if (dma_fence_context->mali_dma_fence_waiters[i]) {
  57                         fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence,
  58                                               &dma_fence_context->mali_dma_fence_waiters[i]->base);
  59                         fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence);
  60                         kfree(dma_fence_context->mali_dma_fence_waiters[i]);
  61                         dma_fence_context->mali_dma_fence_waiters[i] = NULL;
  62                 }
  63         }
  64
  65         if (NULL != dma_fence_context->mali_dma_fence_waiters)
  66                 kfree(dma_fence_context->mali_dma_fence_waiters);
  67
  68         dma_fence_context->mali_dma_fence_waiters = NULL;
  69         dma_fence_context->num_dma_fence_waiter = 0;
  70 }
  71
  72 static void mali_dma_fence_context_work_func(struct work_struct *work_handle)
  73 {
  74         struct mali_dma_fence_context *dma_fence_context;
  75
  76         MALI_DEBUG_ASSERT_POINTER(work_handle);
  77
  78         dma_fence_context = container_of(work_handle, struct mali_dma_fence_context, work_handle);
  79
  80         dma_fence_context->cb_func(dma_fence_context->pp_job_ptr);
  81 }
  82
  83 static void mali_dma_fence_callback(struct fence *fence, struct fence_cb *cb)
  84 {
  85         struct mali_dma_fence_waiter *dma_fence_waiter = NULL;
  86         struct mali_dma_fence_context *dma_fence_context = NULL;
  87
  88         MALI_DEBUG_ASSERT_POINTER(fence);
  89         MALI_DEBUG_ASSERT_POINTER(cb);
  90
  91         MALI_IGNORE(fence);
  92
  93         dma_fence_waiter = container_of(cb, struct mali_dma_fence_waiter, base);
  94         dma_fence_context = dma_fence_waiter->parent;
  95
  96         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
  97
  98         if (atomic_dec_and_test(&dma_fence_context->count))
  99                 schedule_work(&dma_fence_context->work_handle);
 100 }
 101
 102 static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct fence *fence)
 103 {
 104         int ret = 0;
 105         struct mali_dma_fence_waiter *dma_fence_waiter;
 106         struct mali_dma_fence_waiter **dma_fence_waiters;
 107
 108         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
 109         MALI_DEBUG_ASSERT_POINTER(fence);
 110
 111         dma_fence_waiters = krealloc(dma_fence_context->mali_dma_fence_waiters,
 112                                      (dma_fence_context->num_dma_fence_waiter + 1)
 113                                      * sizeof(struct mali_dma_fence_waiter *),
 114                                      GFP_KERNEL);
 115
 116         if (NULL == dma_fence_waiters) {
 117                 MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to realloc the dma fence waiters.\n"));
 118                 return _MALI_OSK_ERR_NOMEM;
 119         }
 120
 121         dma_fence_context->mali_dma_fence_waiters = dma_fence_waiters;
 122
 123         dma_fence_waiter = kzalloc(sizeof(struct mali_dma_fence_waiter), GFP_KERNEL);
 124
 125         if (NULL == dma_fence_waiter) {
 126                 MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create mali dma fence waiter.\n"));
 127                 return _MALI_OSK_ERR_NOMEM;
 128         }
 129
 130         fence_get(fence);
 131
 132         dma_fence_waiter->fence = fence;
 133         dma_fence_waiter->parent = dma_fence_context;
 134         atomic_inc(&dma_fence_context->count);
 135
 136         ret = fence_add_callback(fence, &dma_fence_waiter->base,
 137                                  mali_dma_fence_callback);
 138         if (0 > ret) {
 139                 fence_put(fence);
 140                 kfree(dma_fence_waiter);
 141                 atomic_dec(&dma_fence_context->count);
 142                 if (-ENOENT == ret) {
 143                         /*-ENOENT if fence has already been signaled, return _MALI_OSK_ERR_OK*/
 144                         return _MALI_OSK_ERR_OK;
 145                 }
 146                 /* Failed to add the fence callback into fence, return _MALI_OSK_ERR_FAULT*/
 147                 MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into fence.\n"));
 148                 return _MALI_OSK_ERR_FAULT;
 149         }
 150
 151         dma_fence_context->mali_dma_fence_waiters[dma_fence_context->num_dma_fence_waiter] = dma_fence_waiter;
 152         dma_fence_context->num_dma_fence_waiter++;
 153
 154         return _MALI_OSK_ERR_OK;
 155 }
 156
 157
 158 struct fence *mali_dma_fence_new(u32  context, u32 seqno)
 159 {
 160         struct fence *fence = NULL;
 161
 162         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 163
 164         if (NULL == fence) {
 165                 MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create dma fence.\n"));
 166                 return fence;
 167         }
 168
 169         fence_init(fence,
 170                    &mali_dma_fence_ops,
 171                    &mali_dma_fence_lock,
 172                    context, seqno);
 173
 174         return fence;
 175 }
 176
 177 void mali_dma_fence_signal_and_put(struct fence **fence)
 178 {
 179         MALI_DEBUG_ASSERT_POINTER(fence);
 180         MALI_DEBUG_ASSERT_POINTER(*fence);
 181
 182         fence_signal(*fence);
 183         fence_put(*fence);
 184         *fence = NULL;
 185 }
 186
 187 void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context,
 188                                  mali_dma_fence_context_callback_func_t  cb_func,
 189                                  void *pp_job_ptr)
 190 {
 191         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
 192
 193         INIT_WORK(&dma_fence_context->work_handle, mali_dma_fence_context_work_func);
 194         atomic_set(&dma_fence_context->count, 1);
 195         dma_fence_context->num_dma_fence_waiter = 0;
 196         dma_fence_context->mali_dma_fence_waiters = NULL;
 197         dma_fence_context->cb_func = cb_func;
 198         dma_fence_context->pp_job_ptr = pp_job_ptr;
 199 }
 200
 201 _mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context,
 202                 struct reservation_object *dma_reservation_object)
 203 {
 204         _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK;
 205         struct fence *exclusive_fence = NULL;
 206         u32 shared_count = 0, i;
 207         struct fence **shared_fences = NULL;
 208
 209         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
 210         MALI_DEBUG_ASSERT_POINTER(dma_reservation_object);
 211
 212         /* Get all the shared/exclusive fences in the reservation object of dma buf*/
 213         ret = reservation_object_get_fences_rcu(dma_reservation_object, &exclusive_fence,
 214                                                 &shared_count, &shared_fences);
 215         if (ret < 0) {
 216                 MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to get  shared or exclusive_fence dma fences from  the reservation object of dma buf.\n"));
 217                 return _MALI_OSK_ERR_FAULT;
 218         }
 219
 220         if (exclusive_fence) {
 221                 ret = mali_dma_fence_add_callback(dma_fence_context, exclusive_fence);
 222                 if (_MALI_OSK_ERR_OK != ret) {
 223                         MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into exclusive fence.\n"));
 224                         mali_dma_fence_context_cleanup(dma_fence_context);
 225                         goto ended;
 226                 }
 227         }
 228
 229
 230         for (i = 0; i < shared_count; i++) {
 231                 ret = mali_dma_fence_add_callback(dma_fence_context, shared_fences[i]);
 232                 if (_MALI_OSK_ERR_OK != ret) {
 233                         MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into shared fence [%d].\n", i));
 234                         mali_dma_fence_context_cleanup(dma_fence_context);
 235                         break;
 236                 }
 237         }
 238
 239 ended:
 240
 241         if (exclusive_fence)
 242                 fence_put(exclusive_fence);
 243
 244         if (shared_fences) {
 245                 for (i = 0; i < shared_count; i++) {
 246                         fence_put(shared_fences[i]);
 247                 }
 248                 kfree(shared_fences);
 249         }
 250
 251         return ret;
 252 }
 253
 254
 255 void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context)
 256 {
 257         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
 258         atomic_set(&dma_fence_context->count, 0);
 259         if (dma_fence_context->work_handle.func) {
 260                 cancel_work_sync(&dma_fence_context->work_handle);
 261         }
 262         mali_dma_fence_context_cleanup(dma_fence_context);
 263 }
 264
 265 void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context)
 266 {
 267         MALI_DEBUG_ASSERT_POINTER(dma_fence_context);
 268
 269         if (atomic_dec_and_test(&dma_fence_context->count))
 270                 schedule_work(&dma_fence_context->work_handle);
 271 }
 272
 273
 274 void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object,
 275                 struct reservation_object **dma_reservation_object_list,
 276                 u32 *num_dma_reservation_object)
 277 {
 278         u32 i;
 279
 280         MALI_DEBUG_ASSERT_POINTER(dma_reservation_object);
 281         MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list);
 282         MALI_DEBUG_ASSERT_POINTER(num_dma_reservation_object);
 283
 284         for (i = 0; i < *num_dma_reservation_object; i++) {
 285                 if (dma_reservation_object_list[i] == dma_reservation_object)
 286                         return;
 287         }
 288
 289         dma_reservation_object_list[*num_dma_reservation_object] = dma_reservation_object;
 290         (*num_dma_reservation_object)++;
 291 }
 292
 293 int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list,
 294                 u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx)
 295 {
 296         u32 i;
 297
 298         struct reservation_object *reservation_object_to_slow_lock = NULL;
 299
 300         MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list);
 301         MALI_DEBUG_ASSERT_POINTER(ww_actx);
 302
 303         ww_acquire_init(ww_actx, &reservation_ww_class);
 304
 305 again:
 306         for (i = 0; i < num_dma_reservation_object; i++) {
 307                 int ret;
 308
 309                 if (dma_reservation_object_list[i] == reservation_object_to_slow_lock) {
 310                         reservation_object_to_slow_lock = NULL;
 311                         continue;
 312                 }
 313
 314                 ret = ww_mutex_lock(&dma_reservation_object_list[i]->lock, ww_actx);
 315
 316                 if (ret < 0) {
 317                         u32  slow_lock_index = i;
 318
 319                         /* unlock all pre locks we have already locked.*/
 320                         while (i > 0) {
 321                                 i--;
 322                                 ww_mutex_unlock(&dma_reservation_object_list[i]->lock);
 323                         }
 324
 325                         if (NULL != reservation_object_to_slow_lock)
 326                                 ww_mutex_unlock(&reservation_object_to_slow_lock->lock);
 327
 328                         if (ret == -EDEADLK) {
 329                                 reservation_object_to_slow_lock = dma_reservation_object_list[slow_lock_index];
 330                                 ww_mutex_lock_slow(&reservation_object_to_slow_lock->lock, ww_actx);
 331                                 goto again;
 332                         }
 333                         ww_acquire_fini(ww_actx);
 334                         MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to lock all dma reservation objects.\n", i));
 335                         return ret;
 336                 }
 337         }
 338
 339         ww_acquire_done(ww_actx);
 340         return 0;
 341 }
 342
 343 void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list,
 344                 u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx)
 345 {
 346         u32 i;
 347
 348         for (i = 0; i < num_dma_reservation_object; i++)
 349                 ww_mutex_unlock(&dma_reservation_object_list[i]->lock);
 350
 351         ww_acquire_fini(ww_actx);
 352 }