3 * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_defs.h
23 * Defintions (types, defines, etcs) common to Kbase. They are placed here to
24 * allow the hierarchy of header files to work.
27 #ifndef _KBASE_DEFS_H_
28 #define _KBASE_DEFS_H_
30 #include <mali_kbase_config.h>
31 #include <mali_base_hwconfig_features.h>
32 #include <mali_base_hwconfig_issues.h>
33 #include <mali_kbase_mem_lowlevel.h>
34 #include <mali_kbase_mmu_hw.h>
35 #include <mali_kbase_mmu_mode.h>
36 #include <mali_kbase_instr.h>
38 #include <linux/atomic.h>
39 #include <linux/mempool.h>
40 #include <linux/slab.h>
41 #include <linux/file.h>
43 #ifdef CONFIG_MALI_FPGA_BUS_LOGGER
44 #include <linux/bus_logger.h>
49 #include <linux/kds.h>
50 #endif /* CONFIG_KDS */
54 #endif /* CONFIG_SYNC */
56 #ifdef CONFIG_DEBUG_FS
57 #include <linux/debugfs.h>
58 #endif /* CONFIG_DEBUG_FS */
60 #ifdef CONFIG_PM_DEVFREQ
61 #include <linux/devfreq.h>
62 #endif /* CONFIG_DEVFREQ */
64 #include <linux/clk.h>
65 #include <linux/regulator/consumer.h>
67 #if defined(CONFIG_PM)
68 #define KBASE_PM_RUNTIME 1
71 /** Enable SW tracing when set */
72 #ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE
73 #define KBASE_TRACE_ENABLE 1
76 #ifndef KBASE_TRACE_ENABLE
77 #ifdef CONFIG_MALI_DEBUG
78 #define KBASE_TRACE_ENABLE 1
80 #define KBASE_TRACE_ENABLE 0
81 #endif /* CONFIG_MALI_DEBUG */
82 #endif /* KBASE_TRACE_ENABLE */
84 /** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */
85 #define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1
88 * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware.
89 * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU
90 * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware
93 #define ZAP_TIMEOUT 1000
95 /** Number of milliseconds before we time out on a GPU soft/hard reset */
96 #define RESET_TIMEOUT 500
99 * Prevent soft-stops from occuring in scheduling situations
101 * This is not due to HW issues, but when scheduling is desired to be more predictable.
103 * Therefore, soft stop may still be disabled due to HW issues.
105 * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context.
107 * @note if not in use, define this value to 0 instead of \#undef'ing it
109 #define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0
112 * Prevent hard-stops from occuring in scheduling situations
114 * This is not due to HW issues, but when scheduling is desired to be more predictable.
116 * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context.
118 * @note if not in use, define this value to 0 instead of \#undef'ing it
120 #define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0
123 * The maximum number of Job Slots to support in the Hardware.
125 * You can optimize this down if your target devices will only ever support a
126 * small number of job slots.
128 #define BASE_JM_MAX_NR_SLOTS 3
131 * The maximum number of Address Spaces to support in the Hardware.
133 * You can optimize this down if your target devices will only ever support a
134 * small number of Address Spaces
136 #define BASE_MAX_NR_AS 16
139 #define MIDGARD_MMU_VA_BITS 48
141 #if MIDGARD_MMU_VA_BITS > 39
142 #define MIDGARD_MMU_TOPLEVEL 0
144 #define MIDGARD_MMU_TOPLEVEL 1
147 #define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR)
149 /** setting in kbase_context::as_nr that indicates it's invalid */
150 #define KBASEP_AS_NR_INVALID (-1)
152 #define KBASE_LOCK_REGION_MAX_SIZE (63)
153 #define KBASE_LOCK_REGION_MIN_SIZE (11)
155 #define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */
156 #define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2)
157 #define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1)
159 #include "mali_kbase_js_defs.h"
160 #include "mali_kbase_hwaccess_defs.h"
162 #define KBASEP_FORCE_REPLAY_DISABLED 0
164 /* Maximum force replay limit when randomization is enabled */
165 #define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16
167 /** Atom has been previously soft-stoppped */
168 #define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1)
169 /** Atom has been previously retried to execute */
170 #define KBASE_KATOM_FLAGS_RERUN (1<<2)
171 #define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)
172 /** Atom has been previously hard-stopped. */
173 #define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
174 /** Atom has caused us to enter disjoint state */
175 #define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
176 /* Atom has fail dependency on same-slot dependency */
177 #define KBASE_KATOM_FLAG_FAIL_PREV (1<<6)
178 /* Atom blocked on cross-slot dependency */
179 #define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
180 /* Atom has fail dependency on cross-slot dependency */
181 #define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
182 /* Atom has been submitted to JSCTX ringbuffers */
183 #define KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED (1<<9)
184 /* Atom is currently holding a context reference */
185 #define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10)
186 /* Atom requires GPU to be in secure mode */
187 #define KBASE_KATOM_FLAG_SECURE (1<<11)
189 /* SW related flags about types of JS_COMMAND action
190 * NOTE: These must be masked off by JS_COMMAND_MASK */
192 /** This command causes a disjoint event */
193 #define JS_COMMAND_SW_CAUSES_DISJOINT 0x100
195 /** Bitmask of all SW related flags */
196 #define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT)
198 #if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
199 #error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks
202 /** Soft-stop command that causes a Disjoint event. This of course isn't
203 * entirely masked off by JS_COMMAND_MASK */
204 #define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
205 (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
207 #define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
209 #ifdef CONFIG_DEBUG_FS
210 struct base_job_fault_event {
213 struct kbase_jd_atom *katom;
214 struct work_struct job_fault_work;
215 struct list_head head;
221 struct kbase_jd_atom_dependency {
222 struct kbase_jd_atom *atom;
227 * @brief The function retrieves a read-only reference to the atom field from
228 * the kbase_jd_atom_dependency structure
230 * @param[in] dep kbase jd atom dependency.
232 * @return readonly reference to dependent ATOM.
234 static inline const struct kbase_jd_atom *const kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
236 LOCAL_ASSERT(dep != NULL);
238 return (const struct kbase_jd_atom * const)(dep->atom);
242 * @brief The function retrieves a read-only reference to the dependency type field from
243 * the kbase_jd_atom_dependency structure
245 * @param[in] dep kbase jd atom dependency.
247 * @return A dependency type value.
249 static inline const u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep)
251 LOCAL_ASSERT(dep != NULL);
253 return dep->dep_type;
257 * @brief Setter macro for dep_atom array entry in kbase_jd_atom
259 * @param[in] dep The kbase jd atom dependency.
260 * @param[in] a The ATOM to be set as a dependency.
261 * @param type The ATOM dependency type to be set.
264 static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep,
265 struct kbase_jd_atom *a, u8 type)
267 struct kbase_jd_atom_dependency *dep;
269 LOCAL_ASSERT(const_dep != NULL);
271 dep = (struct kbase_jd_atom_dependency *)const_dep;
274 dep->dep_type = type;
278 * @brief Setter macro for dep_atom array entry in kbase_jd_atom
280 * @param[in] dep The kbase jd atom dependency to be cleared.
283 static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep)
285 struct kbase_jd_atom_dependency *dep;
287 LOCAL_ASSERT(const_dep != NULL);
289 dep = (struct kbase_jd_atom_dependency *)const_dep;
292 dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
295 enum kbase_atom_gpu_rb_state {
296 /* Atom is not currently present in slot ringbuffer */
297 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
298 /* Atom is in slot ringbuffer but is blocked on a previous atom */
299 KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
300 /* Atom is in slot ringbuffer but is waiting for cores to become
302 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
303 /* Atom is in slot ringbuffer but is blocked on affinity */
304 KBASE_ATOM_GPU_RB_WAITING_AFFINITY,
305 /* Atom is in slot ringbuffer but is waiting for secure mode switch */
306 KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE,
307 /* Atom is in slot ringbuffer and ready to run */
308 KBASE_ATOM_GPU_RB_READY,
309 /* Atom is in slot ringbuffer and has been submitted to the GPU */
310 KBASE_ATOM_GPU_RB_SUBMITTED,
311 /* Atom must be returned to JS as soon as it reaches the head of the
312 * ringbuffer due to a previous failure */
313 KBASE_ATOM_GPU_RB_RETURN_TO_JS
316 struct kbase_ext_res {
318 struct kbase_mem_phy_alloc *alloc;
321 struct kbase_jd_atom {
322 struct work_struct work;
323 ktime_t start_timestamp;
324 u64 time_spent_us; /**< Total time spent on the GPU in microseconds */
326 struct base_jd_udata udata;
327 struct kbase_context *kctx;
329 struct list_head dep_head[2];
330 struct list_head dep_item[2];
331 const struct kbase_jd_atom_dependency dep[2];
334 struct kbase_ext_res *extres;
339 enum kbase_atom_coreref_state coreref_state;
341 struct list_head node;
342 struct kds_resource_set *kds_rset;
343 bool kds_dep_satisfied;
344 #endif /* CONFIG_KDS */
346 struct sync_fence *fence;
347 struct sync_fence_waiter sync_waiter;
348 #endif /* CONFIG_SYNC */
350 /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */
351 enum base_jd_event_code event_code;
352 base_jd_core_req core_req; /**< core requirements */
353 /** Job Slot to retry submitting to if submission from IRQ handler failed
355 * NOTE: see if this can be unified into the another member e.g. the event */
356 int retry_submit_on_slot;
358 union kbasep_js_policy_job_info sched_info;
359 /* JS atom priority with respect to other atoms on its kctx. */
362 int poking; /* BASE_HW_ISSUE_8316 */
364 wait_queue_head_t completed;
365 enum kbase_jd_atom_state status;
366 #ifdef CONFIG_GPU_TRACEPOINTS
369 /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */
374 /* Number of times this atom has been retried. Used by replay soft job.
378 enum kbase_atom_gpu_rb_state gpu_rb_state;
380 u64 need_cache_flush_cores_retained;
384 /* Pointer to atom that this atom has cross-slot dependency on */
385 struct kbase_jd_atom *x_pre_dep;
386 /* Pointer to atom that has cross-slot dependency on this atom */
387 struct kbase_jd_atom *x_post_dep;
390 struct kbase_jd_atom_backend backend;
391 #ifdef CONFIG_DEBUG_FS
392 struct base_job_fault_event fault_event;
396 static inline bool kbase_jd_katom_is_secure(const struct kbase_jd_atom *katom)
398 return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_SECURE);
402 * Theory of operations:
404 * Atom objects are statically allocated within the context structure.
406 * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set.
409 #define KBASE_JD_DEP_QUEUE_SIZE 256
411 struct kbase_jd_context {
413 struct kbasep_js_kctx_info sched_info;
414 struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];
416 /** Tracks all job-dispatch jobs. This includes those not tracked by
417 * the scheduler: 'not ready to run' and 'dependency-only' jobs. */
420 /** Waitq that reflects whether there are no jobs (including SW-only
421 * dependency jobs). This is set when no jobs are present on the ctx,
422 * and clear when there are jobs.
424 * @note: Job Dispatcher knows about more jobs than the Job Scheduler:
425 * the Job Scheduler is unaware of jobs that are blocked on dependencies,
426 * and SW-only dependency jobs.
428 * This waitq can be waited upon to find out when the context jobs are all
429 * done/cancelled (including those that might've been blocked on
430 * dependencies) - and so, whether it can be terminated. However, it should
431 * only be terminated once it is neither present in the policy-queue (see
432 * kbasep_js_policy_try_evict_ctx() ) nor the run-pool (see
433 * kbasep_js_kctx_info::ctx::is_scheduled).
435 * Since the waitq is only set under kbase_jd_context::lock,
436 * the waiter should also briefly obtain and drop kbase_jd_context::lock to
437 * guarentee that the setter has completed its work on the kbase_context
439 * This must be updated atomically with:
440 * - kbase_jd_context::job_nr */
441 wait_queue_head_t zero_jobs_wait;
443 /** Job Done workqueue. */
444 struct workqueue_struct *job_done_wq;
448 size_t tb_wrap_offset;
451 struct kds_callback kds_cb;
452 #endif /* CONFIG_KDS */
453 #ifdef CONFIG_GPU_TRACEPOINTS
458 struct kbase_device_info {
462 /** Poking state for BASE_HW_ISSUE_8316 */
464 KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0,
465 KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1
468 /** Poking state for BASE_HW_ISSUE_8316 */
469 typedef u32 kbase_as_poke_state;
471 struct kbase_mmu_setup {
477 * Important: Our code makes assumptions that a struct kbase_as structure is always at
478 * kbase_device->as[number]. This is used to recover the containing
479 * struct kbase_device from a struct kbase_as structure.
481 * Therefore, struct kbase_as structures must not be allocated anywhere else.
486 struct workqueue_struct *pf_wq;
487 struct work_struct work_pagefault;
488 struct work_struct work_busfault;
489 enum kbase_mmu_fault_type fault_type;
492 struct mutex transaction_mutex;
494 struct kbase_mmu_setup current_setup;
496 /* BASE_HW_ISSUE_8316 */
497 struct workqueue_struct *poke_wq;
498 struct work_struct poke_work;
499 /** Protected by kbasep_js_device_data::runpool_irq::lock */
501 /** Protected by kbasep_js_device_data::runpool_irq::lock */
502 kbase_as_poke_state poke_state;
503 struct hrtimer poke_timer;
506 static inline int kbase_as_has_bus_fault(struct kbase_as *as)
508 return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS;
511 static inline int kbase_as_has_page_fault(struct kbase_as *as)
513 return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE;
516 struct kbasep_mem_device {
517 atomic_t used_pages; /* Tracks usage of OS shared memory. Updated
518 when OS memory is allocated/freed. */
522 #define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X
524 enum kbase_trace_code {
525 /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
526 * THIS MUST BE USED AT THE START OF THE ENUM */
527 #define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X)
528 #include "mali_kbase_trace_defs.h"
529 #undef KBASE_TRACE_CODE_MAKE_CODE
530 /* Comma on its own, to extend the list */
532 /* Must be the last in the enum */
533 KBASE_TRACE_CODE_COUNT
536 #define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0)
537 #define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1)
540 struct timespec timestamp;
548 unsigned long info_val;
555 /** Event IDs for the power management framework.
557 * Any of these events might be missed, so they should not be relied upon to
558 * find the precise state of the GPU at a particular time in the
559 * trace. Overall, we should get a high percentage of these events for
560 * statisical purposes, and so a few missing should not be a problem */
561 enum kbase_timeline_pm_event {
562 /* helper for tests */
563 KBASEP_TIMELINE_PM_EVENT_FIRST,
565 /** Event reserved for backwards compatibility with 'init' events */
566 KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST,
568 /** The power state of the device has changed.
570 * Specifically, the device has reached a desired or available state.
572 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED,
574 /** The GPU is becoming active.
576 * This event is sent when the first context is about to use the GPU.
578 KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE,
580 /** The GPU is becoming idle.
582 * This event is sent when the last context has finished using the GPU.
584 KBASE_TIMELINE_PM_EVENT_GPU_IDLE,
586 /** Event reserved for backwards compatibility with 'policy_change'
588 KBASE_TIMELINE_PM_EVENT_RESERVED_4,
590 /** Event reserved for backwards compatibility with 'system_suspend'
592 KBASE_TIMELINE_PM_EVENT_RESERVED_5,
594 /** Event reserved for backwards compatibility with 'system_resume'
596 KBASE_TIMELINE_PM_EVENT_RESERVED_6,
598 /** The job scheduler is requesting to power up/down cores.
600 * This event is sent when:
601 * - powered down cores are needed to complete a job
602 * - powered up cores are not needed anymore
604 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE,
606 KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE,
609 #ifdef CONFIG_MALI_TRACE_TIMELINE
610 struct kbase_trace_kctx_timeline {
611 atomic_t jd_atoms_in_flight;
615 struct kbase_trace_kbdev_timeline {
616 /* Note: strictly speaking, not needed, because it's in sync with
617 * kbase_device::jm_slots[]::submitted_nr
619 * But it's kept as an example of how to add global timeline tracking
622 * The caller must hold kbasep_js_device_data::runpool_irq::lock when
624 u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS];
626 /* Last UID for each PM event */
627 atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1];
628 /* Counter for generating PM event UIDs */
629 atomic_t pm_event_uid_counter;
631 * L2 transition state - true indicates that the transition is ongoing
632 * Expected to be protected by pm.power_change_lock */
633 bool l2_transitioning;
635 #endif /* CONFIG_MALI_TRACE_TIMELINE */
638 struct kbasep_kctx_list_element {
639 struct list_head link;
640 struct kbase_context *kctx;
644 * Data stored per device for power management.
646 * This structure contains data for the power management framework. There is one
647 * instance of this structure per device in the system.
649 struct kbase_pm_device_data {
651 * The lock protecting Power Management structures accessed outside of
654 * This lock must also be held whenever the GPU is being powered on or
659 /** The reference count of active contexts on this device. */
661 /** Flag indicating suspending/suspended */
663 /* Wait queue set when active_count == 0 */
664 wait_queue_head_t zero_active_count_wait;
667 * Bit masks identifying the available shader cores that are specified
668 * via sysfs. One mask per job slot.
670 u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS];
671 u64 debug_core_mask_all;
674 * Lock protecting the power state of the device.
676 * This lock must be held when accessing the shader_available_bitmap,
677 * tiler_available_bitmap, l2_available_bitmap, shader_inuse_bitmap and
678 * tiler_inuse_bitmap fields of kbase_device, and the ca_in_transition
679 * and shader_poweroff_pending fields of kbase_pm_device_data. It is
680 * also held when the hardware power registers are being written to, to
681 * ensure that two threads do not conflict over the power transitions
682 * that the hardware should make.
684 spinlock_t power_change_lock;
687 * Callback for initializing the runtime power management.
689 * @param kbdev The kbase device
691 * @return 0 on success, else error code
693 int (*callback_power_runtime_init)(struct kbase_device *kbdev);
696 * Callback for terminating the runtime power management.
698 * @param kbdev The kbase device
700 void (*callback_power_runtime_term)(struct kbase_device *kbdev);
702 /* Time in milliseconds between each dvfs sample */
705 /* Period of GPU poweroff timer */
706 ktime_t gpu_poweroff_time;
708 /* Number of ticks of GPU poweroff timer before shader is powered off */
709 int poweroff_shader_ticks;
711 /* Number of ticks of GPU poweroff timer before GPU is powered off */
712 int poweroff_gpu_ticks;
714 struct kbase_pm_backend_data backend;
718 * struct kbase_secure_ops - Platform specific functions for GPU secure mode
720 * @secure_mode_enable: Callback to enable secure mode on the GPU
721 * @secure_mode_disable: Callback to disable secure mode on the GPU
723 struct kbase_secure_ops {
725 * secure_mode_enable() - Enable secure mode on the GPU
726 * @kbdev: The kbase device
728 * Return: 0 on success, non-zero on error
730 int (*secure_mode_enable)(struct kbase_device *kbdev);
733 * secure_mode_disable() - Disable secure mode on the GPU
734 * @kbdev: The kbase device
736 * Return: 0 on success, non-zero on error
738 int (*secure_mode_disable)(struct kbase_device *kbdev);
743 * struct kbase_mem_pool - Page based memory pool for kctx/kbdev
744 * @kbdev: Kbase device where memory is used
745 * @cur_size: Number of free pages currently in the pool (may exceed @max_size
746 * in some corner cases)
747 * @max_size: Maximum number of free pages in the pool
748 * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size
750 * @page_list: List of free pages in the pool
751 * @reclaim: Shrinker for kernel reclaim of free pages
752 * @next_pool: Pointer to next pool where pages can be allocated when this pool
753 * is empty. Pages will spill over to the next pool when this pool
754 * is full. Can be NULL if there is no next pool.
756 struct kbase_mem_pool {
757 struct kbase_device *kbdev;
760 spinlock_t pool_lock;
761 struct list_head page_list;
762 struct shrinker reclaim;
764 struct kbase_mem_pool *next_pool;
768 #define DEVNAME_SIZE 16
770 struct kbase_device {
771 s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS];
778 struct list_head entry;
780 unsigned int kbase_group_error;
781 struct miscdevice mdev;
789 #ifdef CONFIG_HAVE_CLK
792 #ifdef CONFIG_REGULATOR
793 struct regulator *regulator;
795 char devname[DEVNAME_SIZE];
797 #ifdef CONFIG_MALI_NO_MALI
799 struct kmem_cache *irq_slab;
800 struct workqueue_struct *irq_workq;
801 atomic_t serving_job_irq;
802 atomic_t serving_gpu_irq;
803 atomic_t serving_mmu_irq;
804 spinlock_t reg_op_lock;
805 #endif /* CONFIG_MALI_NO_MALI */
807 struct kbase_pm_device_data pm;
808 struct kbasep_js_device_data js_data;
809 struct kbase_mem_pool mem_pool;
810 struct kbasep_mem_device memdev;
811 struct kbase_mmu_mode const *mmu_mode;
813 struct kbase_as as[BASE_MAX_NR_AS];
815 spinlock_t mmu_mask_change;
817 struct kbase_gpu_props gpu_props;
819 /** List of SW workarounds for HW issues */
820 unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG];
821 /** List of features available */
822 unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG];
824 /* Bitmaps of cores that are currently in use (running jobs).
825 * These should be kept up to date by the job scheduler.
827 * pm.power_change_lock should be held when accessing these members.
829 * kbase_pm_check_transitions_nolock() should be called when bits are
830 * cleared to update the power management system and allow transitions to
832 u64 shader_inuse_bitmap;
834 /* Refcount for cores in use */
835 u32 shader_inuse_cnt[64];
837 /* Bitmaps of cores the JS needs for jobs ready to run */
838 u64 shader_needed_bitmap;
840 /* Refcount for cores needed */
841 u32 shader_needed_cnt[64];
845 u32 tiler_needed_cnt;
847 /* struct for keeping track of the disjoint information
849 * The state is > 0 if the GPU is in a disjoint state. Otherwise 0
850 * The count is the number of disjoint events that have occurred on the GPU
857 /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */
860 /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be
861 * submitted to these cores. These are updated by the power management code. The job scheduler should avoid
862 * submitting new jobs to any cores that are not marked as available.
864 * pm.power_change_lock should be held when accessing these members.
866 u64 shader_available_bitmap;
867 u64 tiler_available_bitmap;
868 u64 l2_available_bitmap;
870 u64 shader_ready_bitmap;
871 u64 shader_transitioning_bitmap;
873 s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */
874 s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */
876 /* Structure used for instrumentation and HW counters dumping */
878 /* The lock should be used when accessing any of the following members */
881 struct kbase_context *kctx;
884 struct kbase_context *suspended_kctx;
885 struct kbase_uk_hwcnt_setup suspended_state;
887 struct kbase_instr_backend backend;
890 struct kbase_vinstr_context *vinstr_ctx;
892 /*value to be written to the irq_throttle register each time an irq is served */
893 atomic_t irq_throttle_cycles;
895 #if KBASE_TRACE_ENABLE
896 spinlock_t trace_lock;
899 struct kbase_trace *trace_rbuf;
902 /* This is used to override the current job scheduler values for
903 * JS_SCHEDULING_PERIOD_NS
905 * JS_SOFT_STOP_TICKS_CL
906 * JS_HARD_STOP_TICKS_SS
907 * JS_HARD_STOP_TICKS_CL
908 * JS_HARD_STOP_TICKS_DUMPING
911 * JS_RESET_TICKS_DUMPING.
913 * These values are set via the js_timeouts sysfs file.
915 u32 js_scheduling_period_ns;
916 int js_soft_stop_ticks;
917 int js_soft_stop_ticks_cl;
918 int js_hard_stop_ticks_ss;
919 int js_hard_stop_ticks_cl;
920 int js_hard_stop_ticks_dumping;
921 int js_reset_ticks_ss;
922 int js_reset_ticks_cl;
923 int js_reset_ticks_dumping;
924 bool js_timeouts_updated;
926 u32 reset_timeout_ms;
928 struct mutex cacheclean_lock;
930 /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */
931 void *platform_context;
933 /* List of kbase_contexts created */
934 struct list_head kctx_list;
935 struct mutex kctx_list_lock;
937 #ifdef CONFIG_MALI_MIDGARD_RT_PM
938 struct delayed_work runtime_pm_workqueue;
941 #ifdef CONFIG_PM_DEVFREQ
942 struct devfreq_dev_profile devfreq_profile;
943 struct devfreq *devfreq;
944 unsigned long current_freq;
945 unsigned long current_voltage;
946 #ifdef CONFIG_DEVFREQ_THERMAL
947 struct devfreq_cooling_device *devfreq_cooling;
951 struct kbase_ipa_context *ipa_ctx;
953 #ifdef CONFIG_MALI_TRACE_TIMELINE
954 struct kbase_trace_kbdev_timeline timeline;
958 * Control for enabling job dump on failure, set when control debugfs
961 bool job_fault_debug;
963 #ifdef CONFIG_DEBUG_FS
964 /* directory for debugfs entries */
965 struct dentry *mali_debugfs_directory;
966 /* Root directory for per context entry */
967 struct dentry *debugfs_ctx_directory;
969 /* failed job dump, used for separate debug process */
970 wait_queue_head_t job_fault_wq;
971 wait_queue_head_t job_fault_resume_wq;
972 struct workqueue_struct *job_fault_resume_workq;
973 struct list_head job_fault_event_list;
974 struct kbase_context *kctx_fault;
976 #if !MALI_CUSTOMER_RELEASE
977 /* Per-device data for register dumping interface */
979 u16 reg_offset; /* Offset of a GPU_CONTROL register to be
980 dumped upon request */
981 } regs_dump_debugfs_data;
982 #endif /* !MALI_CUSTOMER_RELEASE */
983 #endif /* CONFIG_DEBUG_FS */
985 /* fbdump profiling controls set by gator */
986 u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX];
989 #if MALI_CUSTOMER_RELEASE == 0
990 /* Number of jobs that are run before a job is forced to fail and
991 * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced
993 int force_replay_limit;
994 /* Count of jobs between forced failures. Incremented on each job. A
995 * job is forced to fail once this is greater than or equal to
996 * force_replay_limit. */
997 int force_replay_count;
998 /* Core requirement for jobs to be failed and replayed. May be zero. */
999 base_jd_core_req force_replay_core_req;
1000 /* true if force_replay_limit should be randomized. The random
1001 * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT.
1003 bool force_replay_random;
1006 /* Total number of created contexts */
1009 struct kbase_hwaccess_data hwaccess;
1011 /* Count of page/bus faults waiting for workqueues to process */
1012 atomic_t faults_pending;
1014 /* true if GPU is powered off or power off operation is in progress */
1015 bool poweroff_pending;
1018 /* defaults for new context created for this device */
1019 u32 infinite_cache_active_default;
1020 size_t mem_pool_max_size_default;
1022 /* system coherency mode */
1023 u32 system_coherency;
1025 /* Secure operations */
1026 struct kbase_secure_ops *secure_ops;
1029 * true when GPU is put into secure mode
1034 * true if secure mode is supported
1036 bool secure_mode_support;
1039 #ifdef CONFIG_MALI_DEBUG
1040 wait_queue_head_t driver_inactive_wait;
1041 bool driver_inactive;
1042 #endif /* CONFIG_MALI_DEBUG */
1044 #ifdef CONFIG_MALI_FPGA_BUS_LOGGER
1046 * Bus logger integration.
1048 struct bus_logger_client *buslogger;
1050 /* Boolean indicating if an IRQ flush during reset is in progress. */
1051 bool irq_reset_flush;
1054 /* JSCTX ringbuffer size must always be a power of 2 */
1055 #define JSCTX_RB_SIZE 256
1056 #define JSCTX_RB_MASK (JSCTX_RB_SIZE-1)
1059 * struct jsctx_rb_entry - Entry in &struct jsctx_rb ring buffer
1062 struct jsctx_rb_entry {
1067 * struct jsctx_rb - JS context atom ring buffer
1068 * @entries: Array of size %JSCTX_RB_SIZE which holds the &struct
1069 * kbase_jd_atom pointers which make up the contents of the ring
1071 * @read_idx: Index into @entries. Indicates the next entry in @entries to
1072 * read, and is incremented when pulling an atom, and decremented
1074 * HW access lock must be held when accessing.
1075 * @write_idx: Index into @entries. Indicates the next entry to use when
1076 * adding atoms into the ring buffer, and is incremented when
1077 * adding a new atom.
1078 * jctx->lock must be held when accessing.
1079 * @running_idx: Index into @entries. Indicates the last valid entry, and is
1080 * incremented when remving atoms from the ring buffer.
1081 * HW access lock must be held when accessing.
1083 * &struct jsctx_rb is a ring buffer of &struct kbase_jd_atom.
1086 struct jsctx_rb_entry entries[JSCTX_RB_SIZE];
1088 u16 read_idx; /* HW access lock must be held when accessing */
1089 u16 write_idx; /* jctx->lock must be held when accessing */
1090 u16 running_idx; /* HW access lock must be held when accessing */
1093 #define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \
1094 (((minor) & 0xFFF) << 8) | \
1097 struct kbase_context {
1099 struct kbase_device *kbdev;
1100 int id; /* System wide unique id */
1101 unsigned long api_version;
1103 struct list_head event_list;
1104 struct mutex event_mutex;
1105 atomic_t event_closed;
1106 struct workqueue_struct *event_workq;
1107 atomic_t event_count;
1111 atomic_t setup_complete;
1112 atomic_t setup_in_progress;
1114 u64 *mmu_teardown_pages;
1116 struct page *aliasing_sink_page;
1118 struct mutex reg_lock; /* To be converted to a rwlock? */
1119 struct rb_root reg_rbtree; /* Red-Black tree of GPU regions (live regions) */
1121 unsigned long cookies;
1122 struct kbase_va_region *pending_regions[BITS_PER_LONG];
1124 wait_queue_head_t event_queue;
1128 struct kbase_jd_context jctx;
1129 atomic_t used_pages;
1130 atomic_t nonmapped_pages;
1132 struct kbase_mem_pool mem_pool;
1134 struct list_head waiting_soft_jobs;
1136 struct list_head waiting_kds_resource;
1138 /** This is effectively part of the Run Pool, because it only has a valid
1139 * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in
1141 * The kbasep_js_device_data::runpool_irq::lock must be held whilst accessing
1144 * If the context relating to this as_nr is required, you must use
1145 * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear
1146 * whilst you're using it. Alternatively, just hold the kbasep_js_device_data::runpool_irq::lock
1147 * to ensure the context doesn't disappear (but this has restrictions on what other locks
1148 * you can take whilst doing this) */
1153 * Flags are in jctx.sched_info.ctx.flags
1154 * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex
1156 * All other flags must be added there */
1157 spinlock_t mm_update_lock;
1158 struct mm_struct *process_mm;
1160 #ifdef CONFIG_MALI_TRACE_TIMELINE
1161 struct kbase_trace_kctx_timeline timeline;
1163 #ifdef CONFIG_DEBUG_FS
1164 /* Content of mem_profile file */
1165 char *mem_profile_data;
1166 /* Size of @c mem_profile_data */
1167 size_t mem_profile_size;
1168 /* Mutex guarding memory profile state */
1169 struct mutex mem_profile_lock;
1170 /* Memory profile file created */
1171 bool mem_profile_initialized;
1172 struct dentry *kctx_dentry;
1174 /* for job fault debug */
1175 unsigned int *reg_dump;
1176 atomic_t job_fault_count;
1177 /* This list will keep the following atoms during the dump
1178 * in the same context
1180 struct list_head job_fault_resume_event_list;
1182 #endif /* CONFIG_DEBUG_FS */
1184 struct jsctx_rb jsctx_rb
1185 [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS];
1187 /* Number of atoms currently pulled from this context */
1188 atomic_t atoms_pulled;
1189 /* Number of atoms currently pulled from this context, per slot */
1190 atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS];
1191 /* true if last kick() caused atoms to be pulled from this context */
1193 /* true if infinite cache is to be enabled for new allocations. Existing
1194 * allocations will not change. bool stored as a u32 per Linux API */
1195 u32 infinite_cache_active;
1196 /* Bitmask of slots that can be pulled from */
1199 /* true if address space assignment is pending */
1202 /* Backend specific data */
1203 struct kbase_context_backend backend;
1205 /* Work structure used for deferred ASID assignment */
1206 struct work_struct work;
1208 /* Only one userspace vinstr client per kbase context */
1209 struct kbase_vinstr_client *vinstr_cli;
1210 struct mutex vinstr_cli_lock;
1212 /* Must hold queue_mutex when accessing */
1215 /* List of completed jobs waiting for events to be posted */
1216 struct list_head completed_jobs;
1217 /* Number of work items currently pending on job_done_wq */
1218 atomic_t work_count;
1221 enum kbase_reg_access_type {
1226 enum kbase_share_attr_bits {
1227 /* (1ULL << 8) bit is reserved */
1228 SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */
1229 SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */
1233 * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent.
1234 * @kbdev: kbase device
1236 * Return: true if the device access are coherent, false if not.
1238 static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev)
1240 if ((kbdev->system_coherency == COHERENCY_ACE_LITE) ||
1241 (kbdev->system_coherency == COHERENCY_ACE))
1247 /* Conversion helpers for setting up high resolution timers */
1248 #define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime((x)*1000000U))
1249 #define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x))
1251 /* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */
1252 #define KBASE_CLEAN_CACHE_MAX_LOOPS 100000
1253 /* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */
1254 #define KBASE_AS_INACTIVE_MAX_LOOPS 100000
1256 /* Maximum number of times a job can be replayed */
1257 #define BASEP_JD_REPLAY_LIMIT 15
1259 #endif /* _KBASE_DEFS_H_ */