drivers/gpu/arm/mali400/mali/common/mali_pp_job.c

   1 /*
   2  * Copyright (C) 2011-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_pp.h"
  12 #include "mali_pp_job.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_list.h"
  15 #include "mali_kernel_common.h"
  16 #include "mali_uk_types.h"
  17 #include "mali_executor.h"
  18 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  19 #include "linux/mali_memory_dma_buf.h"
  20 #endif
  21 #include "mali_memory_swap_alloc.h"
  22 #include "mali_scheduler.h"
  23
  24 static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER;   /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
  25 static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER;   /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
  26 static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */
  27 static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
  28 static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
  29
  30 void mali_pp_job_initialize(void)
  31 {
  32         _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0);
  33 }
  34
  35 void mali_pp_job_terminate(void)
  36 {
  37         _mali_osk_atomic_term(&pp_counter_per_sub_job_count);
  38 }
  39
  40 struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session,
  41                                        _mali_uk_pp_start_job_s __user *uargs, u32 id)
  42 {
  43         struct mali_pp_job *job;
  44         u32 perf_counter_flag;
  45
  46         job = _mali_osk_calloc(1, sizeof(struct mali_pp_job));
  47         if (NULL != job) {
  48
  49                 _mali_osk_list_init(&job->list);
  50                 _mali_osk_list_init(&job->session_fb_lookup_list);
  51                 _mali_osk_atomic_inc(&session->number_of_pp_jobs);
  52
  53                 if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) {
  54                         goto fail;
  55                 }
  56
  57                 if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) {
  58                         MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n"));
  59                         goto fail;
  60                 }
  61
  62                 if (!mali_pp_job_use_no_notification(job)) {
  63                         job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s));
  64                         if (NULL == job->finished_notification) goto fail;
  65                 }
  66
  67                 perf_counter_flag = mali_pp_job_get_perf_counter_flag(job);
  68
  69                 /* case when no counters came from user space
  70                  * so pass the debugfs / DS-5 provided global ones to the job object */
  71                 if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
  72                       (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
  73                         u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count);
  74
  75                         /* These counters apply for all virtual jobs, and where no per sub job counter is specified */
  76                         job->uargs.perf_counter_src0 = pp_counter_src0;
  77                         job->uargs.perf_counter_src1 = pp_counter_src1;
  78
  79                         /* We only copy the per sub job array if it is enabled with at least one counter */
  80                         if (0 < sub_job_count) {
  81                                 job->perf_counter_per_sub_job_count = sub_job_count;
  82                                 _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0));
  83                                 _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1));
  84                         }
  85                 }
  86
  87                 job->session = session;
  88                 job->id = id;
  89
  90                 job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1;
  91                 job->pid = _mali_osk_get_pid();
  92                 job->tid = _mali_osk_get_tid();
  93
  94                 _mali_osk_atomic_init(&job->sub_jobs_completed, 0);
  95                 _mali_osk_atomic_init(&job->sub_job_errors, 0);
  96                 job->swap_status = MALI_NO_SWAP_IN;
  97                 job->user_notification = MALI_FALSE;
  98                 job->num_pp_cores_in_virtual = 0;
  99
 100                 if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) {
 101                         MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n"));
 102                         goto fail;
 103                 }
 104
 105                 if (job->uargs.num_memory_cookies > 0) {
 106                         u32 size;
 107                         u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies;
 108
 109                         size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies);
 110
 111                         job->memory_cookies = _mali_osk_malloc(size);
 112                         if (NULL == job->memory_cookies) {
 113                                 MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size));
 114                                 goto fail;
 115                         }
 116
 117                         if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) {
 118                                 MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size));
 119                                 goto fail;
 120                         }
 121                 }
 122
 123                 if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) {
 124                         /* Not a valid job. */
 125                         goto fail;
 126                 }
 127
 128                 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job);
 129                 mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
 130
 131                 mali_mem_swap_in_pages(job);
 132
 133                 return job;
 134         }
 135
 136 fail:
 137         if (NULL != job) {
 138                 mali_pp_job_delete(job);
 139         }
 140
 141         return NULL;
 142 }
 143
 144 void mali_pp_job_delete(struct mali_pp_job *job)
 145 {
 146         struct mali_session_data *session;
 147
 148         MALI_DEBUG_ASSERT_POINTER(job);
 149         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list));
 150         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list));
 151
 152         session = mali_pp_job_get_session(job);
 153         MALI_DEBUG_ASSERT_POINTER(session);
 154
 155         if (NULL != job->memory_cookies) {
 156 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
 157                 /* Unmap buffers attached to job */
 158                 mali_dma_buf_unmap_job(job);
 159 #endif
 160                 if (MALI_NO_SWAP_IN != job->swap_status) {
 161                         mali_mem_swap_out_pages(job);
 162                 }
 163
 164                 _mali_osk_free(job->memory_cookies);
 165         }
 166
 167         if (job->user_notification) {
 168                 mali_scheduler_return_pp_job_to_user(job,
 169                                                      job->num_pp_cores_in_virtual);
 170         }
 171
 172         if (NULL != job->finished_notification) {
 173                 _mali_osk_notification_delete(job->finished_notification);
 174         }
 175
 176         _mali_osk_atomic_term(&job->sub_jobs_completed);
 177         _mali_osk_atomic_term(&job->sub_job_errors);
 178         _mali_osk_atomic_dec(&session->number_of_pp_jobs);
 179         _mali_osk_free(job);
 180
 181         _mali_osk_wait_queue_wake_up(session->wait_queue);
 182 }
 183
 184 void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list)
 185 {
 186         struct mali_pp_job *iter;
 187         struct mali_pp_job *tmp;
 188
 189         MALI_DEBUG_ASSERT_POINTER(job);
 190         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
 191
 192         /* Find position in list/queue where job should be added. */
 193         _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list,
 194                                             struct mali_pp_job, list) {
 195                 /* job should be started after iter if iter is in progress. */
 196                 if (0 < iter->sub_jobs_started) {
 197                         break;
 198                 }
 199
 200                 /*
 201                  * job should be started after iter if it has a higher
 202                  * job id. A span is used to handle job id wrapping.
 203                  */
 204                 if ((mali_pp_job_get_id(job) -
 205                      mali_pp_job_get_id(iter)) <
 206                     MALI_SCHEDULER_JOB_ID_SPAN) {
 207                         break;
 208                 }
 209         }
 210
 211         _mali_osk_list_add(&job->list, &iter->list);
 212 }
 213
 214
 215 u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job)
 216 {
 217         /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
 218         if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
 219                 return job->uargs.perf_counter_src0;
 220         }
 221
 222         /* Use per sub job counter if enabled... */
 223         if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) {
 224                 return job->perf_counter_per_sub_job_src0[sub_job];
 225         }
 226
 227         /* ...else default to global job counter */
 228         return job->uargs.perf_counter_src0;
 229 }
 230
 231 u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job)
 232 {
 233         /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
 234         if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
 235                 /* Virtual jobs always use the global job counter */
 236                 return job->uargs.perf_counter_src1;
 237         }
 238
 239         /* Use per sub job counter if enabled... */
 240         if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) {
 241                 return job->perf_counter_per_sub_job_src1[sub_job];
 242         }
 243
 244         /* ...else default to global job counter */
 245         return job->uargs.perf_counter_src1;
 246 }
 247
 248 void mali_pp_job_set_pp_counter_global_src0(u32 counter)
 249 {
 250         pp_counter_src0 = counter;
 251 }
 252
 253 void mali_pp_job_set_pp_counter_global_src1(u32 counter)
 254 {
 255         pp_counter_src1 = counter;
 256 }
 257
 258 void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter)
 259 {
 260         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 261
 262         if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) {
 263                 /* increment count since existing counter was disabled */
 264                 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
 265         }
 266
 267         if (MALI_HW_CORE_NO_COUNTER == counter) {
 268                 /* decrement count since new counter is disabled */
 269                 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
 270         }
 271
 272         /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
 273
 274         pp_counter_per_sub_job_src0[sub_job] = counter;
 275 }
 276
 277 void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter)
 278 {
 279         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 280
 281         if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) {
 282                 /* increment count since existing counter was disabled */
 283                 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
 284         }
 285
 286         if (MALI_HW_CORE_NO_COUNTER == counter) {
 287                 /* decrement count since new counter is disabled */
 288                 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
 289         }
 290
 291         /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
 292
 293         pp_counter_per_sub_job_src1[sub_job] = counter;
 294 }
 295
 296 u32 mali_pp_job_get_pp_counter_global_src0(void)
 297 {
 298         return pp_counter_src0;
 299 }
 300
 301 u32 mali_pp_job_get_pp_counter_global_src1(void)
 302 {
 303         return pp_counter_src1;
 304 }
 305
 306 u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job)
 307 {
 308         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 309         return pp_counter_per_sub_job_src0[sub_job];
 310 }
 311
 312 u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job)
 313 {
 314         MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
 315         return pp_counter_per_sub_job_src1[sub_job];
 316 }