3 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
23 #include <mali_malisw.h>
25 #include <mali_kbase_debug.h>
29 #include <linux/atomic.h>
30 #include <linux/highmem.h>
31 #include <linux/hrtimer.h>
32 #include <linux/ktime.h>
33 #include <linux/list.h>
34 #include <linux/mm_types.h>
35 #include <linux/mutex.h>
36 #include <linux/rwsem.h>
37 #include <linux/sched.h>
38 #include <linux/slab.h>
39 #include <linux/spinlock.h>
40 #include <linux/vmalloc.h>
41 #include <linux/wait.h>
42 #include <linux/workqueue.h>
44 #include "mali_base_kernel.h"
45 #include <mali_kbase_uku.h>
46 #include <mali_kbase_linux.h>
49 * Include mali_kbase_defs.h first as this provides types needed by other local
52 #include "mali_kbase_defs.h"
54 #include "mali_kbase_context.h"
55 #include "mali_kbase_strings.h"
56 #include "mali_kbase_mem_lowlevel.h"
57 #include "mali_kbase_trace_timeline.h"
58 #include "mali_kbase_js.h"
59 #include "mali_kbase_mem.h"
60 #include "mali_kbase_utility.h"
61 #include "mali_kbase_gpu_memory_debugfs.h"
62 #include "mali_kbase_mem_profile_debugfs.h"
63 #include "mali_kbase_debug_job_fault.h"
64 #include "mali_kbase_jd_debugfs.h"
65 #include "mali_kbase_gpuprops.h"
66 #include "mali_kbase_jm.h"
67 #include "mali_kbase_vinstr.h"
68 #include "mali_kbase_ipa.h"
69 #ifdef CONFIG_GPU_TRACEPOINTS
70 #include <trace/events/gpu.h>
73 * @page page_base_kernel_main Kernel-side Base (KBase) APIs
75 * The Kernel-side Base (KBase) APIs are divided up as follows:
76 * - @subpage page_kbase_js_policy
80 * @defgroup base_kbase_api Kernel-side Base (KBase) APIs
83 struct kbase_device *kbase_device_alloc(void);
85 * note: configuration attributes member of kbdev needs to have
86 * been setup before calling kbase_device_init
90 * API to acquire device list semaphore and return pointer
91 * to the device list head
93 const struct list_head *kbase_dev_list_get(void);
94 /* API to release the device list semaphore */
95 void kbase_dev_list_put(const struct list_head *dev_list);
97 int kbase_device_init(struct kbase_device * const kbdev);
98 void kbase_device_term(struct kbase_device *kbdev);
99 void kbase_device_free(struct kbase_device *kbdev);
100 int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature);
102 /* Needed for gator integration and for reporting vsync information */
103 struct kbase_device *kbase_find_device(int minor);
104 void kbase_release_device(struct kbase_device *kbdev);
106 void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value);
108 u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control);
110 struct kbase_context *
111 kbase_create_context(struct kbase_device *kbdev, bool is_compat);
112 void kbase_destroy_context(struct kbase_context *kctx);
114 int kbase_jd_init(struct kbase_context *kctx);
115 void kbase_jd_exit(struct kbase_context *kctx);
116 #ifdef BASE_LEGACY_UK6_SUPPORT
117 int kbase_jd_submit(struct kbase_context *kctx,
118 const struct kbase_uk_job_submit *submit_data,
121 int kbase_jd_submit(struct kbase_context *kctx,
122 const struct kbase_uk_job_submit *submit_data);
126 * kbase_jd_done_worker - Handle a job completion
127 * @data: a &struct work_struct
129 * This function requeues the job from the runpool (if it was soft-stopped or
130 * removed from NEXT registers).
132 * Removes it from the system if it finished/failed/was cancelled.
134 * Resolves dependencies to add dependent jobs to the context, potentially
135 * starting them if necessary (which may add more references to the context)
137 * Releases the reference to the context from the no-longer-running job.
139 * Handles retrying submission outside of IRQ context if it failed from within
142 void kbase_jd_done_worker(struct work_struct *data);
144 void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp,
145 kbasep_js_atom_done_code done_code);
146 void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
147 void kbase_jd_zap_context(struct kbase_context *kctx);
148 bool jd_done_nolock(struct kbase_jd_atom *katom,
149 struct list_head *completed_jobs_ctx);
150 void kbase_jd_free_external_resources(struct kbase_jd_atom *katom);
151 bool jd_submit_atom(struct kbase_context *kctx,
152 const struct base_jd_atom_v2 *user_atom,
153 struct kbase_jd_atom *katom);
154 void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom);
156 void kbase_job_done(struct kbase_device *kbdev, u32 done);
158 void kbase_gpu_cacheclean(struct kbase_device *kbdev,
159 struct kbase_jd_atom *katom);
161 * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms
163 * @kctx: Pointer to context to check.
164 * @katom: Pointer to priority atom.
166 * Atoms from @kctx on the same job slot as @katom, which have lower priority
167 * than @katom will be soft stopped and put back in the queue, so that atoms
168 * with higher priority can run.
170 * The hwaccess_lock must be held when calling this function.
172 void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
173 struct kbase_jd_atom *katom);
175 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
176 struct kbase_jd_atom *target_katom);
177 void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
178 struct kbase_jd_atom *target_katom, u32 sw_flags);
179 void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
180 struct kbase_jd_atom *target_katom);
181 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
182 base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom);
183 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
184 struct kbase_jd_atom *target_katom);
186 void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event);
187 int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent);
188 int kbase_event_pending(struct kbase_context *ctx);
189 int kbase_event_init(struct kbase_context *kctx);
190 void kbase_event_close(struct kbase_context *kctx);
191 void kbase_event_cleanup(struct kbase_context *kctx);
192 void kbase_event_wakeup(struct kbase_context *kctx);
194 int kbase_process_soft_job(struct kbase_jd_atom *katom);
195 int kbase_prepare_soft_job(struct kbase_jd_atom *katom);
196 void kbase_finish_soft_job(struct kbase_jd_atom *katom);
197 void kbase_cancel_soft_job(struct kbase_jd_atom *katom);
198 void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev);
199 void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom);
200 void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom);
201 int kbase_soft_event_update(struct kbase_context *kctx,
203 unsigned char new_status);
205 bool kbase_replay_process(struct kbase_jd_atom *katom);
207 void kbasep_soft_job_timeout_worker(unsigned long data);
208 void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt);
210 /* api used internally for register access. Contains validation and tracing */
211 void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value);
212 int kbase_device_trace_buffer_install(
213 struct kbase_context *kctx, u32 *tb, size_t size);
214 void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx);
216 /* api to be ported per OS, only need to do the raw register access */
217 void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value);
218 u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset);
220 void kbasep_as_do_poke(struct work_struct *work);
222 /** Returns the name associated with a Mali exception code
224 * This function is called from the interrupt handler when a GPU fault occurs.
225 * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN.
227 * @param[in] kbdev The kbase device that the GPU fault occurred from.
228 * @param[in] exception_code exception code
229 * @return name associated with the exception code
231 const char *kbase_exception_name(struct kbase_device *kbdev,
235 * Check whether a system suspend is in progress, or has already been suspended
237 * The caller should ensure that either kbdev->pm.active_count_lock is held, or
238 * a dmb was executed recently (to ensure the value is most
239 * up-to-date). However, without a lock the value could change afterwards.
241 * @return false if a suspend is not in progress
242 * @return !=false otherwise
244 static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev)
246 return kbdev->pm.suspending;
250 * Return the atom's ID, as was originally supplied by userspace in
251 * base_jd_atom_v2::atom_number
253 static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom)
257 KBASE_DEBUG_ASSERT(kctx);
258 KBASE_DEBUG_ASSERT(katom);
259 KBASE_DEBUG_ASSERT(katom->kctx == kctx);
261 result = katom - &kctx->jctx.atoms[0];
262 KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT);
267 * kbase_jd_atom_from_id - Return the atom structure for the given atom ID
268 * @kctx: Context pointer
269 * @id: ID of atom to retrieve
271 * Return: Pointer to struct kbase_jd_atom associated with the supplied ID
273 static inline struct kbase_jd_atom *kbase_jd_atom_from_id(
274 struct kbase_context *kctx, int id)
276 return &kctx->jctx.atoms[id];
280 * Initialize the disjoint state
282 * The disjoint event count and state are both set to zero.
284 * Disjoint functions usage:
286 * The disjoint event count should be incremented whenever a disjoint event occurs.
288 * There are several cases which are regarded as disjoint behavior. Rather than just increment
289 * the counter during disjoint events we also increment the counter when jobs may be affected
290 * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state.
292 * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying
293 * (as part of the replay workaround). Increasing the disjoint state also increases the count of
296 * The disjoint state is then used to increase the count of disjoint events during job submission
297 * and job completion. Any atom submitted or completed while the disjoint state is greater than
298 * zero is regarded as a disjoint event.
300 * The disjoint event counter is also incremented immediately whenever a job is soft stopped
301 * and during context creation.
303 * @param kbdev The kbase device
305 void kbase_disjoint_init(struct kbase_device *kbdev);
308 * Increase the count of disjoint events
309 * called when a disjoint event has happened
311 * @param kbdev The kbase device
313 void kbase_disjoint_event(struct kbase_device *kbdev);
316 * Increase the count of disjoint events only if the GPU is in a disjoint state
318 * This should be called when something happens which could be disjoint if the GPU
319 * is in a disjoint state. The state refcount keeps track of this.
321 * @param kbdev The kbase device
323 void kbase_disjoint_event_potential(struct kbase_device *kbdev);
326 * Returns the count of disjoint events
328 * @param kbdev The kbase device
329 * @return the count of disjoint events
331 u32 kbase_disjoint_event_get(struct kbase_device *kbdev);
334 * Increment the refcount state indicating that the GPU is in a disjoint state.
336 * Also Increment the disjoint event count (calls @ref kbase_disjoint_event)
337 * eventually after the disjoint state has completed @ref kbase_disjoint_state_down
340 * @param kbdev The kbase device
342 void kbase_disjoint_state_up(struct kbase_device *kbdev);
345 * Decrement the refcount state
347 * Also Increment the disjoint event count (calls @ref kbase_disjoint_event)
349 * Called after @ref kbase_disjoint_state_up once the disjoint state is over
351 * @param kbdev The kbase device
353 void kbase_disjoint_state_down(struct kbase_device *kbdev);
356 * If a job is soft stopped and the number of contexts is >= this value
357 * it is reported as a disjoint event
359 #define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2
361 #if !defined(UINT64_MAX)
362 #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
365 #if KBASE_TRACE_ENABLE
366 void kbasep_trace_debugfs_init(struct kbase_device *kbdev);
368 #ifndef CONFIG_MALI_SYSTEM_TRACE
369 /** Add trace values about a job-slot
371 * @note Any functions called through this macro will still be evaluated in
372 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
373 * functions called to get the parameters supplied to this macro must:
374 * - be static or static inline
375 * - must just return 0 and have no other statements present in the body.
377 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \
378 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
379 KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0)
381 /** Add trace values about a job-slot, with info
383 * @note Any functions called through this macro will still be evaluated in
384 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
385 * functions called to get the parameters supplied to this macro must:
386 * - be static or static inline
387 * - must just return 0 and have no other statements present in the body.
389 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \
390 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
391 KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val)
393 /** Add trace values about a ctx refcount
395 * @note Any functions called through this macro will still be evaluated in
396 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
397 * functions called to get the parameters supplied to this macro must:
398 * - be static or static inline
399 * - must just return 0 and have no other statements present in the body.
401 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \
402 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
403 KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0)
404 /** Add trace values about a ctx refcount, and info
406 * @note Any functions called through this macro will still be evaluated in
407 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
408 * functions called to get the parameters supplied to this macro must:
409 * - be static or static inline
410 * - must just return 0 and have no other statements present in the body.
412 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \
413 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
414 KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val)
416 /** Add trace values (no slot or refcount)
418 * @note Any functions called through this macro will still be evaluated in
419 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
420 * functions called to get the parameters supplied to this macro must:
421 * - be static or static inline
422 * - must just return 0 and have no other statements present in the body.
424 #define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \
425 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
428 /** Clear the trace */
429 #define KBASE_TRACE_CLEAR(kbdev) \
430 kbasep_trace_clear(kbdev)
432 /** Dump the slot trace */
433 #define KBASE_TRACE_DUMP(kbdev) \
434 kbasep_trace_dump(kbdev)
436 /** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */
437 void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val);
438 /** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */
439 void kbasep_trace_clear(struct kbase_device *kbdev);
440 #else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
441 /* Dispatch kbase trace events as system trace events */
442 #include <mali_linux_kbase_trace.h>
443 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
444 trace_mali_##code(jobslot, 0)
446 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
447 trace_mali_##code(jobslot, info_val)
449 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
450 trace_mali_##code(refcount, 0)
452 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
453 trace_mali_##code(refcount, info_val)
455 #define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\
456 trace_mali_##code(gpu_addr, info_val)
458 #define KBASE_TRACE_CLEAR(kbdev)\
463 #define KBASE_TRACE_DUMP(kbdev)\
469 #endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
471 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
477 CSTD_UNUSED(gpu_addr);\
478 CSTD_UNUSED(jobslot);\
481 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
487 CSTD_UNUSED(gpu_addr);\
488 CSTD_UNUSED(jobslot);\
489 CSTD_UNUSED(info_val);\
493 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
499 CSTD_UNUSED(gpu_addr);\
500 CSTD_UNUSED(refcount);\
504 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
510 CSTD_UNUSED(gpu_addr);\
511 CSTD_UNUSED(info_val);\
515 #define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\
519 CSTD_UNUSED(subcode);\
526 #define KBASE_TRACE_CLEAR(kbdev)\
531 #define KBASE_TRACE_DUMP(kbdev)\
536 #endif /* KBASE_TRACE_ENABLE */
537 /** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */
538 void kbasep_trace_dump(struct kbase_device *kbdev);
540 #ifdef CONFIG_MALI_DEBUG
542 * kbase_set_driver_inactive - Force driver to go inactive
543 * @kbdev: Device pointer
544 * @inactive: true if driver should go inactive, false otherwise
546 * Forcing the driver inactive will cause all future IOCTLs to wait until the
547 * driver is made active again. This is intended solely for the use of tests
548 * which require that no jobs are running while the test executes.
550 void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive);
551 #endif /* CONFIG_MALI_DEBUG */
554 #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI)
556 /* kbase_io_history_init - initialize data struct for register access history
558 * @kbdev The register history to initialize
559 * @n The number of register accesses that the buffer could hold
561 * @return 0 if successfully initialized, failure otherwise
563 int kbase_io_history_init(struct kbase_io_history *h, u16 n);
565 /* kbase_io_history_term - uninit all resources for the register access history
567 * @h The register history to terminate
569 void kbase_io_history_term(struct kbase_io_history *h);
571 /* kbase_io_history_dump - print the register history to the kernel ring buffer
573 * @kbdev Pointer to kbase_device containing the register history to dump
575 void kbase_io_history_dump(struct kbase_device *kbdev);
578 * kbase_io_history_resize - resize the register access history buffer.
580 * @h: Pointer to a valid register history to resize
581 * @new_size: Number of accesses the buffer could hold
583 * A successful resize will clear all recent register accesses.
584 * If resizing fails for any reason (e.g., could not allocate memory, invalid
585 * buffer size) then the original buffer will be kept intact.
587 * @return 0 if the buffer was resized, failure otherwise
589 int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size);
591 #else /* CONFIG_DEBUG_FS */
593 #define kbase_io_history_init(...) ((int)0)
595 #define kbase_io_history_term CSTD_NOP
597 #define kbase_io_history_dump CSTD_NOP
599 #define kbase_io_history_resize CSTD_NOP
601 #endif /* CONFIG_DEBUG_FS */