drivers/gpu/arm/mali400/mali/common/mali_gp_job.c

   1 /*
   2  * Copyright (C) 2011-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_gp_job.h"
  12 #include "mali_osk.h"
  13 #include "mali_osk_list.h"
  14 #include "mali_uk_types.h"
  15 #include "mali_memory_virtual.h"
  16 #include "mali_memory_defer_bind.h"
  17
  18 static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER;      /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
  19 static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER;           /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
  20 static void _mali_gp_del_varying_allocations(struct mali_gp_job *job);
  21
  22
  23 static int _mali_gp_add_varying_allocations(struct mali_session_data *session,
  24                 struct mali_gp_job *job,
  25                 u32 *alloc,
  26                 u32 num)
  27 {
  28         int i = 0;
  29         struct mali_gp_allocation_node *alloc_node;
  30         mali_mem_allocation *mali_alloc = NULL;
  31         struct mali_vma_node *mali_vma_node = NULL;
  32
  33         for (i = 0 ; i < num ; i++) {
  34                 MALI_DEBUG_ASSERT(alloc[i]);
  35                 alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node));
  36                 if (alloc_node) {
  37                         INIT_LIST_HEAD(&alloc_node->node);
  38                         /* find mali allocation structure by vaddress*/
  39                         mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0);
  40
  41                         if (likely(mali_vma_node)) {
  42                                 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
  43                                 MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start);
  44                         } else {
  45                                 MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num));
  46                                 _mali_osk_free(alloc_node);
  47                                 goto fail;
  48                         }
  49                         alloc_node->alloc = mali_alloc;
  50                         /* add to gp job varying alloc list*/
  51                         list_move(&alloc_node->node, &job->varying_alloc);
  52                 } else
  53                         goto fail;
  54         }
  55
  56         return 0;
  57 fail:
  58         MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n"));
  59         _mali_gp_del_varying_allocations(job);
  60         return -1;
  61 }
  62
  63
  64 static void _mali_gp_del_varying_allocations(struct mali_gp_job *job)
  65 {
  66         struct mali_gp_allocation_node *alloc_node, *tmp_node;
  67
  68         list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) {
  69                 list_del(&alloc_node->node);
  70                 kfree(alloc_node);
  71         }
  72         INIT_LIST_HEAD(&job->varying_alloc);
  73 }
  74
  75 struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker)
  76 {
  77         struct mali_gp_job *job;
  78         u32 perf_counter_flag;
  79         u32 __user *memory_list = NULL;
  80         struct mali_gp_allocation_node *alloc_node, *tmp_node;
  81         _mali_uk_gp_start_job_s copy_of_uargs;
  82
  83         job = _mali_osk_calloc(1, sizeof(struct mali_gp_job));
  84         if (NULL != job) {
  85                 job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s));
  86                 if (NULL == job->finished_notification) {
  87                         goto fail3;
  88                 }
  89
  90                 job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s));
  91                 if (NULL == job->oom_notification) {
  92                         goto fail2;
  93                 }
  94
  95                 if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) {
  96                         goto fail1;
  97                 }
  98
  99                 perf_counter_flag = mali_gp_job_get_perf_counter_flag(job);
 100
 101                 /* case when no counters came from user space
 102                  * so pass the debugfs / DS-5 provided global ones to the job object */
 103                 if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
 104                       (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
 105                         mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0());
 106                         mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1());
 107                 }
 108
 109                 _mali_osk_list_init(&job->list);
 110                 job->session = session;
 111                 job->id = id;
 112                 job->heap_current_addr = job->uargs.frame_registers[4];
 113                 job->perf_counter_value0 = 0;
 114                 job->perf_counter_value1 = 0;
 115                 job->pid = _mali_osk_get_pid();
 116                 job->tid = _mali_osk_get_tid();
 117
 118
 119                 INIT_LIST_HEAD(&job->varying_alloc);
 120                 INIT_LIST_HEAD(&job->vary_todo);
 121                 job->dmem = NULL;
 122
 123                 if (job->uargs.deferred_mem_num > session->allocation_mgr.mali_allocation_num) {
 124                         MALI_PRINT_ERROR(("Mali GP job: The number of  varying buffer to defer bind  is invalid !\n"));
 125                         goto fail1;
 126                 }
 127
 128                 /* add varying allocation list*/
 129                 if (job->uargs.deferred_mem_num > 0) {
 130                         /* copy varying list from user space*/
 131                         job->varying_list = _mali_osk_calloc(1, sizeof(u32) * job->uargs.deferred_mem_num);
 132                         if (!job->varying_list) {
 133                                 MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", job->uargs.deferred_mem_num));
 134                                 goto fail1;
 135                         }
 136
 137                         if (0 != _mali_osk_copy_from_user(&copy_of_uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) {
 138                                 goto fail1;
 139                         }
 140                         memory_list = (u32 __user *)(uintptr_t)copy_of_uargs.deferred_mem_list;
 141
 142                         if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32) * job->uargs.deferred_mem_num)) {
 143                                 MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n"));
 144                                 goto fail;
 145                         }
 146
 147                         if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list,
 148                                         job->uargs.deferred_mem_num))) {
 149                                 MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n"));
 150                                 goto fail;
 151                         }
 152
 153                         /* do preparetion for each allocation */
 154                         list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) {
 155                                 if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo, &job->required_varying_memsize))) {
 156                                         MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n"));
 157                                         goto fail;
 158                                 }
 159                         }
 160
 161                         _mali_gp_del_varying_allocations(job);
 162
 163                         /* bind varying here, to avoid memory latency issue. */
 164                         {
 165                                 struct mali_defer_mem_block dmem_block;
 166
 167                                 INIT_LIST_HEAD(&dmem_block.free_pages);
 168                                 atomic_set(&dmem_block.num_free_pages, 0);
 169
 170                                 if (mali_mem_prepare_mem_for_job(job, &dmem_block)) {
 171                                         MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n"));
 172                                         goto fail;
 173                                 }
 174                                 if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job, &dmem_block)) {
 175                                         MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job));
 176                                         goto fail;
 177                                 }
 178                         }
 179
 180                         if (job->uargs.varying_memsize > MALI_UK_BIG_VARYING_SIZE) {
 181                                 job->big_job = 1;
 182                         }
 183                 }
 184                 job->pp_tracker = pp_tracker;
 185                 if (NULL != job->pp_tracker) {
 186                         /* Take a reference on PP job's tracker that will be released when the GP
 187                            job is done. */
 188                         mali_timeline_system_tracker_get(session->timeline_system, pp_tracker);
 189                 }
 190
 191                 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job);
 192                 mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
 193
 194                 return job;
 195         } else {
 196                 MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n"));
 197                 return NULL;
 198         }
 199
 200
 201 fail:
 202         _mali_osk_free(job->varying_list);
 203         /* Handle allocate fail here, free all varying node */
 204         {
 205                 struct mali_backend_bind_list *bkn, *bkn_tmp;
 206                 list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) {
 207                         list_del(&bkn->node);
 208                         _mali_osk_free(bkn);
 209                 }
 210         }
 211 fail1:
 212         _mali_osk_notification_delete(job->oom_notification);
 213 fail2:
 214         _mali_osk_notification_delete(job->finished_notification);
 215 fail3:
 216         _mali_osk_free(job);
 217         return NULL;
 218 }
 219
 220 void mali_gp_job_delete(struct mali_gp_job *job)
 221 {
 222         struct mali_backend_bind_list *bkn, *bkn_tmp;
 223         MALI_DEBUG_ASSERT_POINTER(job);
 224         MALI_DEBUG_ASSERT(NULL == job->pp_tracker);
 225         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list));
 226         _mali_osk_free(job->varying_list);
 227
 228         /* Handle allocate fail here, free all varying node */
 229         list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) {
 230                 list_del(&bkn->node);
 231                 _mali_osk_free(bkn);
 232         }
 233
 234         mali_mem_defer_dmem_free(job);
 235
 236         /* de-allocate the pre-allocated oom notifications */
 237         if (NULL != job->oom_notification) {
 238                 _mali_osk_notification_delete(job->oom_notification);
 239                 job->oom_notification = NULL;
 240         }
 241         if (NULL != job->finished_notification) {
 242                 _mali_osk_notification_delete(job->finished_notification);
 243                 job->finished_notification = NULL;
 244         }
 245
 246         _mali_osk_free(job);
 247 }
 248
 249 void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list)
 250 {
 251         struct mali_gp_job *iter;
 252         struct mali_gp_job *tmp;
 253
 254         MALI_DEBUG_ASSERT_POINTER(job);
 255         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
 256
 257         /* Find position in list/queue where job should be added. */
 258         _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list,
 259                                             struct mali_gp_job, list) {
 260
 261                 /* A span is used to handle job ID wrapping. */
 262                 bool job_is_after = (mali_gp_job_get_id(job) -
 263                                      mali_gp_job_get_id(iter)) <
 264                                     MALI_SCHEDULER_JOB_ID_SPAN;
 265
 266                 if (job_is_after) {
 267                         break;
 268                 }
 269         }
 270
 271         _mali_osk_list_add(&job->list, &iter->list);
 272 }
 273
 274 u32 mali_gp_job_get_gp_counter_src0(void)
 275 {
 276         return gp_counter_src0;
 277 }
 278
 279 void mali_gp_job_set_gp_counter_src0(u32 counter)
 280 {
 281         gp_counter_src0 = counter;
 282 }
 283
 284 u32 mali_gp_job_get_gp_counter_src1(void)
 285 {
 286         return gp_counter_src1;
 287 }
 288
 289 void mali_gp_job_set_gp_counter_src1(u32 counter)
 290 {
 291         gp_counter_src1 = counter;
 292 }
 293
 294 mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success)
 295 {
 296         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 297
 298         MALI_DEBUG_ASSERT_POINTER(job);
 299
 300         if (NULL != job->pp_tracker) {
 301                 schedule_mask |= mali_timeline_system_tracker_put(job->session->timeline_system, job->pp_tracker, MALI_FALSE == success);
 302                 job->pp_tracker = NULL;
 303         }
 304
 305         return schedule_mask;
 306 }