3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
23 #include <malisw/mali_malisw.h>
25 #include <mali_kbase_debug.h>
29 #include <linux/atomic.h>
30 #include <linux/highmem.h>
31 #include <linux/hrtimer.h>
32 #include <linux/ktime.h>
33 #include <linux/list.h>
34 #include <linux/mm_types.h>
35 #include <linux/mutex.h>
36 #include <linux/rwsem.h>
37 #include <linux/sched.h>
38 #include <linux/slab.h>
39 #include <linux/spinlock.h>
40 #include <linux/vmalloc.h>
41 #include <linux/wait.h>
42 #include <linux/workqueue.h>
44 #include "mali_base_kernel.h"
45 #include <mali_kbase_uku.h>
46 #include <mali_kbase_linux.h>
48 #include "mali_kbase_pm.h"
49 #include "mali_kbase_mem_lowlevel.h"
50 #include "mali_kbase_defs.h"
51 #include "mali_kbase_trace_timeline.h"
52 #include "mali_kbase_js.h"
53 #include "mali_kbase_mem.h"
54 #include "mali_kbase_security.h"
55 #include "mali_kbase_utility.h"
56 #include "mali_kbase_gpu_memory_debugfs.h"
57 #include "mali_kbase_mem_profile_debugfs.h"
58 #include "mali_kbase_jd_debugfs.h"
59 #include "mali_kbase_cpuprops.h"
60 #include "mali_kbase_gpuprops.h"
61 #ifdef CONFIG_GPU_TRACEPOINTS
62 #include <trace/events/gpu.h>
65 * @page page_base_kernel_main Kernel-side Base (KBase) APIs
67 * The Kernel-side Base (KBase) APIs are divided up as follows:
68 * - @subpage page_kbase_js_policy
72 * @defgroup base_kbase_api Kernel-side Base (KBase) APIs
75 struct kbase_device *kbase_device_alloc(void);
77 * note: configuration attributes member of kbdev needs to have
78 * been setup before calling kbase_device_init
82 * API to acquire device list semaphone and return pointer
83 * to the device list head
85 const struct list_head *kbase_dev_list_get(void);
86 /* API to release the device list semaphore */
87 void kbase_dev_list_put(const struct list_head *dev_list);
89 mali_error kbase_device_init(struct kbase_device * const kbdev);
90 void kbase_device_term(struct kbase_device *kbdev);
91 void kbase_device_free(struct kbase_device *kbdev);
92 int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature);
93 struct kbase_device *kbase_find_device(int minor); /* Only needed for gator integration */
94 void kbase_release_device(struct kbase_device *kbdev);
96 void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value);
98 u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control);
101 * Ensure that all IRQ handlers have completed execution
103 * @param kbdev The kbase device
105 void kbase_synchronize_irqs(struct kbase_device *kbdev);
106 void kbase_synchronize_irqs(struct kbase_device *kbdev);
108 struct kbase_context *kbase_create_context(struct kbase_device *kbdev);
109 void kbase_destroy_context(struct kbase_context *kctx);
110 mali_error kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags);
112 mali_error kbase_instr_hwcnt_setup(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup);
113 mali_error kbase_instr_hwcnt_enable(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup);
114 mali_error kbase_instr_hwcnt_disable(struct kbase_context *kctx);
115 mali_error kbase_instr_hwcnt_clear(struct kbase_context *kctx);
116 mali_error kbase_instr_hwcnt_dump(struct kbase_context *kctx);
117 mali_error kbase_instr_hwcnt_dump_irq(struct kbase_context *kctx);
118 mali_bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, mali_bool * const success);
119 void kbase_instr_hwcnt_suspend(struct kbase_device *kbdev);
120 void kbase_instr_hwcnt_resume(struct kbase_device *kbdev);
122 void kbasep_cache_clean_worker(struct work_struct *data);
123 void kbase_clean_caches_done(struct kbase_device *kbdev);
126 * The GPU has completed performance count sampling successfully.
128 void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev);
130 mali_error kbase_jd_init(struct kbase_context *kctx);
131 void kbase_jd_exit(struct kbase_context *kctx);
132 #ifdef BASE_LEGACY_UK6_SUPPORT
133 mali_error kbase_jd_submit(struct kbase_context *kctx,
134 const struct kbase_uk_job_submit *submit_data,
137 mali_error kbase_jd_submit(struct kbase_context *kctx,
138 const struct kbase_uk_job_submit *submit_data);
140 void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp,
141 kbasep_js_atom_done_code done_code);
142 void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
143 void kbase_jd_zap_context(struct kbase_context *kctx);
144 mali_bool jd_done_nolock(struct kbase_jd_atom *katom);
145 void kbase_jd_free_external_resources(struct kbase_jd_atom *katom);
146 mali_bool jd_submit_atom(struct kbase_context *kctx,
147 const struct base_jd_atom_v2 *user_atom,
148 struct kbase_jd_atom *katom);
150 mali_error kbase_job_slot_init(struct kbase_device *kbdev);
151 void kbase_job_slot_halt(struct kbase_device *kbdev);
152 void kbase_job_slot_term(struct kbase_device *kbdev);
153 void kbase_job_done(struct kbase_device *kbdev, u32 done);
154 void kbase_job_zap_context(struct kbase_context *kctx);
156 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
157 struct kbase_jd_atom *target_katom);
158 void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
159 struct kbase_jd_atom *target_katom, u32 sw_flags);
160 void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
161 struct kbase_jd_atom *target_katom);
162 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
163 u16 core_reqs, struct kbase_jd_atom *target_katom);
164 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
165 struct kbase_jd_atom *target_katom);
167 void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event);
168 int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent);
169 int kbase_event_pending(struct kbase_context *ctx);
170 mali_error kbase_event_init(struct kbase_context *kctx);
171 void kbase_event_close(struct kbase_context *kctx);
172 void kbase_event_cleanup(struct kbase_context *kctx);
173 void kbase_event_wakeup(struct kbase_context *kctx);
175 int kbase_process_soft_job(struct kbase_jd_atom *katom);
176 mali_error kbase_prepare_soft_job(struct kbase_jd_atom *katom);
177 void kbase_finish_soft_job(struct kbase_jd_atom *katom);
178 void kbase_cancel_soft_job(struct kbase_jd_atom *katom);
179 void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev);
181 bool kbase_replay_process(struct kbase_jd_atom *katom);
183 /* api used internally for register access. Contains validation and tracing */
184 void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, struct kbase_context *kctx);
185 u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, struct kbase_context *kctx);
186 void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value);
187 void kbase_device_trace_buffer_install(struct kbase_context *kctx, u32 *tb, size_t size);
188 void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx);
190 /* api to be ported per OS, only need to do the raw register access */
191 void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value);
192 u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset);
194 void kbasep_as_do_poke(struct work_struct *work);
196 /** Report a GPU fault.
198 * This function is called from the interrupt handler when a GPU fault occurs.
199 * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN.
201 * @param kbdev The kbase device that the GPU fault occurred from.
202 * @param multiple Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS was also set
204 void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple);
206 /** Kill all jobs that are currently running from a context
208 * This is used in response to a page fault to remove all jobs from the faulting context from the hardware.
210 * @param kctx The context to kill jobs from
212 void kbase_job_kill_jobs_from_context(struct kbase_context *kctx);
215 * GPU interrupt handler
217 * This function is called from the interrupt handler when a GPU irq is to be handled.
219 * @param kbdev The kbase device to handle an IRQ for
220 * @param val The value of the GPU IRQ status register which triggered the call
222 void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val);
225 * Prepare for resetting the GPU.
226 * This function just soft-stops all the slots to ensure that as many jobs as possible are saved.
228 * The function returns a boolean which should be interpreted as follows:
229 * - MALI_TRUE - Prepared for reset, kbase_reset_gpu should be called.
230 * - MALI_FALSE - Another thread is performing a reset, kbase_reset_gpu should not be called.
232 * @return See description
234 mali_bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev);
237 * Pre-locked version of @a kbase_prepare_to_reset_gpu.
239 * Identical to @a kbase_prepare_to_reset_gpu, except that the
240 * kbasep_js_device_data::runpool_irq::lock is externally locked.
242 * @see kbase_prepare_to_reset_gpu
244 mali_bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev);
248 * This function should be called after kbase_prepare_to_reset_gpu iff it returns MALI_TRUE.
249 * It should never be called without a corresponding call to kbase_prepare_to_reset_gpu.
251 * After this function is called (or not called if kbase_prepare_to_reset_gpu returned MALI_FALSE),
252 * the caller should wait for kbdev->reset_waitq to be signalled to know when the reset has completed.
254 void kbase_reset_gpu(struct kbase_device *kbdev);
257 * Pre-locked version of @a kbase_reset_gpu.
259 * Identical to @a kbase_reset_gpu, except that the
260 * kbasep_js_device_data::runpool_irq::lock is externally locked.
262 * @see kbase_reset_gpu
264 void kbase_reset_gpu_locked(struct kbase_device *kbdev);
266 /** Returns the name associated with a Mali exception code
268 * @param[in] exception_code exception code
269 * @return name associated with the exception code
271 const char *kbase_exception_name(u32 exception_code);
274 * Check whether a system suspend is in progress, or has already been suspended
276 * The caller should ensure that either kbdev->pm.active_count_lock is held, or
277 * a dmb was executed recently (to ensure the value is most
278 * up-to-date). However, without a lock the value could change afterwards.
280 * @return MALI_FALSE if a suspend is not in progress
281 * @return !=MALI_FALSE otherwise
283 static INLINE mali_bool kbase_pm_is_suspending(struct kbase_device *kbdev) {
284 return kbdev->pm.suspending;
288 * Return the atom's ID, as was originally supplied by userspace in
289 * base_jd_atom_v2::atom_number
291 static INLINE int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom)
294 KBASE_DEBUG_ASSERT(kctx);
295 KBASE_DEBUG_ASSERT(katom);
296 KBASE_DEBUG_ASSERT(katom->kctx == kctx);
298 result = katom - &kctx->jctx.atoms[0];
299 KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT);
304 * Initialize the disjoint state
306 * The disjoint event count and state are both set to zero.
308 * Disjoint functions usage:
310 * The disjoint event count should be incremented whenever a disjoint event occurs.
312 * There are several cases which are regarded as disjoint behavior. Rather than just increment
313 * the counter during disjoint events we also increment the counter when jobs may be affected
314 * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state.
316 * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying
317 * (as part of the replay workaround). Increasing the disjoint state also increases the count of
320 * The disjoint state is then used to increase the count of disjoint events during job submission
321 * and job completion. Any atom submitted or completed while the disjoint state is greater than
322 * zero is regarded as a disjoint event.
324 * The disjoint event counter is also incremented immediately whenever a job is soft stopped
325 * and during context creation.
327 * @param kbdev The kbase device
329 void kbase_disjoint_init(struct kbase_device *kbdev);
332 * Increase the count of disjoint events
333 * called when a disjoint event has happened
335 * @param kbdev The kbase device
337 void kbase_disjoint_event(struct kbase_device *kbdev);
340 * Increase the count of disjoint events only if the GPU is in a disjoint state
342 * This should be called when something happens which could be disjoint if the GPU
343 * is in a disjoint state. The state refcount keeps track of this.
345 * @param kbdev The kbase device
347 void kbase_disjoint_event_potential(struct kbase_device *kbdev);
350 * Returns the count of disjoint events
352 * @param kbdev The kbase device
353 * @return the count of disjoint events
355 u32 kbase_disjoint_event_get(struct kbase_device *kbdev);
358 * Increment the refcount state indicating that the GPU is in a disjoint state.
360 * Also Increment the disjoint event count (calls @ref kbase_disjoint_event)
361 * eventually after the disjoint state has completed @ref kbase_disjoint_state_down
364 * @param kbdev The kbase device
366 void kbase_disjoint_state_up(struct kbase_device *kbdev);
369 * Decrement the refcount state
371 * Also Increment the disjoint event count (calls @ref kbase_disjoint_event)
373 * Called after @ref kbase_disjoint_state_up once the disjoint state is over
375 * @param kbdev The kbase device
377 void kbase_disjoint_state_down(struct kbase_device *kbdev);
380 * If a job is soft stopped and the number of contexts is >= this value
381 * it is reported as a disjoint event
383 #define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2
385 #if KBASE_TRACE_ENABLE
386 #ifndef CONFIG_MALI_SYSTEM_TRACE
387 /** Add trace values about a job-slot
389 * @note Any functions called through this macro will still be evaluated in
390 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
391 * functions called to get the parameters supplied to this macro must:
392 * - be static or static inline
393 * - must just return 0 and have no other statements present in the body.
395 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \
396 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
397 KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0)
399 /** Add trace values about a job-slot, with info
401 * @note Any functions called through this macro will still be evaluated in
402 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
403 * functions called to get the parameters supplied to this macro must:
404 * - be static or static inline
405 * - must just return 0 and have no other statements present in the body.
407 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \
408 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
409 KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val)
411 /** Add trace values about a ctx refcount
413 * @note Any functions called through this macro will still be evaluated in
414 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
415 * functions called to get the parameters supplied to this macro must:
416 * - be static or static inline
417 * - must just return 0 and have no other statements present in the body.
419 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \
420 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
421 KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0)
422 /** Add trace values about a ctx refcount, and info
424 * @note Any functions called through this macro will still be evaluated in
425 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
426 * functions called to get the parameters supplied to this macro must:
427 * - be static or static inline
428 * - must just return 0 and have no other statements present in the body.
430 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \
431 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
432 KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val)
434 /** Add trace values (no slot or refcount)
436 * @note Any functions called through this macro will still be evaluated in
437 * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
438 * functions called to get the parameters supplied to this macro must:
439 * - be static or static inline
440 * - must just return 0 and have no other statements present in the body.
442 #define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \
443 kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
446 /** Clear the trace */
447 #define KBASE_TRACE_CLEAR(kbdev) \
448 kbasep_trace_clear(kbdev)
450 /** Dump the slot trace */
451 #define KBASE_TRACE_DUMP(kbdev) \
452 kbasep_trace_dump(kbdev)
454 /** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */
455 void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val);
456 /** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */
457 void kbasep_trace_clear(struct kbase_device *kbdev);
458 #else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
459 /* Dispatch kbase trace events as system trace events */
460 #include <mali_linux_kbase_trace.h>
461 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
462 trace_mali_##code(jobslot, 0)
464 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
465 trace_mali_##code(jobslot, info_val)
467 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
468 trace_mali_##code(refcount, 0)
470 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
471 trace_mali_##code(refcount, info_val)
473 #define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\
474 trace_mali_##code(gpu_addr, info_val)
476 #define KBASE_TRACE_CLEAR(kbdev)\
481 #define KBASE_TRACE_DUMP(kbdev)\
487 #endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
489 #define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
495 CSTD_UNUSED(gpu_addr);\
496 CSTD_UNUSED(jobslot);\
499 #define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
505 CSTD_UNUSED(gpu_addr);\
506 CSTD_UNUSED(jobslot);\
507 CSTD_UNUSED(info_val);\
511 #define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
517 CSTD_UNUSED(gpu_addr);\
518 CSTD_UNUSED(refcount);\
522 #define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
528 CSTD_UNUSED(gpu_addr);\
529 CSTD_UNUSED(info_val);\
533 #define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\
537 CSTD_UNUSED(subcode);\
544 #define KBASE_TRACE_CLEAR(kbdev)\
549 #define KBASE_TRACE_DUMP(kbdev)\
554 #endif /* KBASE_TRACE_ENABLE */
555 /** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */
556 void kbasep_trace_dump(struct kbase_device *kbdev);