drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c

   1 /**
   2  * Copyright (C) 2010-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11
  12 /**
  13  * @file mali_kernel_linux.c
  14  * Implementation of the Linux device driver entrypoints
  15  */
  16 #include "../platform/rk/custom_log.h"
  17 #include "../platform/rk/rk_ext.h"
  18
  19 #include <linux/module.h>   /* kernel module definitions */
  20 #include <linux/fs.h>       /* file system operations */
  21 #include <linux/cdev.h>     /* character device definitions */
  22 #include <linux/mm.h>       /* memory manager definitions */
  23 #include <linux/mali/mali_utgard_ioctl.h>
  24 #include <linux/version.h>
  25 #include <linux/device.h>
  26 #include "mali_kernel_license.h"
  27 #include <linux/platform_device.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/bug.h>
  30 #include <linux/of.h>
  31 #include <linux/clk.h>
  32 #include <linux/regulator/consumer.h>
  33
  34 #include <linux/mali/mali_utgard.h>
  35 #include "mali_kernel_common.h"
  36 #include "mali_session.h"
  37 #include "mali_kernel_core.h"
  38 #include "mali_osk.h"
  39 #include "mali_kernel_linux.h"
  40 #include "mali_ukk.h"
  41 #include "mali_ukk_wrappers.h"
  42 #include "mali_kernel_sysfs.h"
  43 #include "mali_pm.h"
  44 #include "mali_kernel_license.h"
  45 #include "mali_memory.h"
  46 #include "mali_memory_dma_buf.h"
  47 #include "mali_memory_manager.h"
  48 #include "mali_memory_swap_alloc.h"
  49 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
  50 #include "mali_profiling_internal.h"
  51 #endif
  52 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
  53 #include "mali_osk_profiling.h"
  54 #include "mali_dvfs_policy.h"
  55
  56 static int is_first_resume = 1;
  57 /*Store the clk and vol for boot/insmod and mali_resume*/
  58 static struct mali_gpu_clk_item mali_gpu_clk[2];
  59 #endif
  60
  61 /* Streamline support for the Mali driver */
  62 #if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING)
  63 /* Ask Linux to create the tracepoints */
  64 #define CREATE_TRACE_POINTS
  65 #include "mali_linux_trace.h"
  66
  67 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event);
  68 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter);
  69 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters);
  70 #endif /* CONFIG_TRACEPOINTS */
  71
  72 #ifdef CONFIG_MALI_DEVFREQ
  73 #include "mali_devfreq.h"
  74 #include "mali_osk_mali.h"
  75
  76 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
  77 #include <linux/pm_opp.h>
  78 #else
  79 /* In 3.13 the OPP include header file, types, and functions were all
  80  * renamed. Use the old filename for the include, and define the new names to
  81  * the old, when an old kernel is detected.
  82  */
  83 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
  84 #include <linux/pm_opp.h>
  85 #else
  86 #include <linux/opp.h>
  87 #endif /* Linux >= 3.13*/
  88 #define dev_pm_opp_of_add_table of_init_opp_table
  89 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
  90 #define dev_pm_opp_of_remove_table of_free_opp_table
  91 #endif /* Linux >= 3.19 */
  92 #endif /* Linux >= 4.4.0 */
  93 #endif
  94
  95 /* from the __malidrv_build_info.c file that is generated during build */
  96 extern const char *__malidrv_build_info(void);
  97
  98 /* Module parameter to control log level */
  99 int mali_debug_level = 2;
 100 module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
 101 MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output");
 102
 103 extern int mali_max_job_runtime;
 104 module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 105 MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what");
 106
 107 extern int mali_l2_max_reads;
 108 module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH);
 109 MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache");
 110
 111 extern unsigned int mali_dedicated_mem_start;
 112 module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH);
 113 MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory.");
 114
 115 extern unsigned int mali_dedicated_mem_size;
 116 module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
 117 MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory.");
 118
 119 extern unsigned int mali_shared_mem_size;
 120 module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
 121 MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory.");
 122
 123 #if defined(CONFIG_MALI400_PROFILING)
 124 extern int mali_boot_profiling;
 125 module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH);
 126 MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization");
 127 #endif
 128
 129 extern int mali_max_pp_cores_group_1;
 130 module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH);
 131 MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group.");
 132
 133 extern int mali_max_pp_cores_group_2;
 134 module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH);
 135 MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only).");
 136
 137 extern unsigned int mali_mem_swap_out_threshold_value;
 138 module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH);
 139 MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver.");
 140
 141 #if defined(CONFIG_MALI_DVFS)
 142 /** the max fps the same as display vsync default 60, can set by module insert parameter */
 143 extern int mali_max_system_fps;
 144 module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 145 MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC.");
 146
 147 /** a lower limit on their desired FPS default 58, can set by module insert parameter*/
 148 extern int mali_desired_fps;
 149 module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 150 MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps");
 151 #endif
 152
 153 #if MALI_ENABLE_CPU_CYCLES
 154 #include <linux/cpumask.h>
 155 #include <linux/timer.h>
 156 #include <asm/smp.h>
 157 static struct timer_list mali_init_cpu_clock_timers[8];
 158 static u32 mali_cpu_clock_last_value[8] = {0,};
 159 #endif
 160
 161 /* Export symbols from common code: mali_user_settings.c */
 162 #include "mali_user_settings_db.h"
 163 EXPORT_SYMBOL(mali_set_user_setting);
 164 EXPORT_SYMBOL(mali_get_user_setting);
 165
 166 static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */
 167
 168 /* This driver only supports one Mali device, and this variable stores this single platform device */
 169 struct platform_device *mali_platform_device = NULL;
 170
 171 /* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */
 172 static struct miscdevice mali_miscdevice = { 0, };
 173
 174 static int mali_miscdevice_register(struct platform_device *pdev);
 175 static void mali_miscdevice_unregister(void);
 176
 177 static int mali_open(struct inode *inode, struct file *filp);
 178 static int mali_release(struct inode *inode, struct file *filp);
 179 #ifdef HAVE_UNLOCKED_IOCTL
 180 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 181 #else
 182 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
 183 #endif
 184
 185 static int mali_probe(struct platform_device *pdev);
 186 static int mali_remove(struct platform_device *pdev);
 187
 188 static int mali_driver_suspend_scheduler(struct device *dev);
 189 static int mali_driver_resume_scheduler(struct device *dev);
 190
 191 #ifdef CONFIG_PM_RUNTIME
 192 static int mali_driver_runtime_suspend(struct device *dev);
 193 static int mali_driver_runtime_resume(struct device *dev);
 194 static int mali_driver_runtime_idle(struct device *dev);
 195 #endif
 196
 197 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 198 #if defined(CONFIG_MALI_DT)
 199 extern int mali_platform_device_init(struct platform_device *device);
 200 extern int mali_platform_device_deinit(struct platform_device *device);
 201 #else
 202 extern int mali_platform_device_register(void);
 203 extern int mali_platform_device_unregister(void);
 204 #endif
 205 #endif
 206
 207 /* Linux power management operations provided by the Mali device driver */
 208 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 209 struct pm_ext_ops mali_dev_ext_pm_ops = {
 210         .base =
 211         {
 212                 .suspend = mali_driver_suspend_scheduler,
 213                 .resume = mali_driver_resume_scheduler,
 214                 .freeze = mali_driver_suspend_scheduler,
 215                 .thaw =   mali_driver_resume_scheduler,
 216         },
 217 };
 218 #else
 219 static const struct dev_pm_ops mali_dev_pm_ops = {
 220 #ifdef CONFIG_PM_RUNTIME
 221         .runtime_suspend = mali_driver_runtime_suspend,
 222         .runtime_resume = mali_driver_runtime_resume,
 223         .runtime_idle = mali_driver_runtime_idle,
 224 #endif
 225         .suspend = mali_driver_suspend_scheduler,
 226         .resume = mali_driver_resume_scheduler,
 227         .freeze = mali_driver_suspend_scheduler,
 228         .thaw = mali_driver_resume_scheduler,
 229         .poweroff = mali_driver_suspend_scheduler,
 230 };
 231 #endif
 232
 233 #ifdef CONFIG_MALI_DT
 234 static struct of_device_id base_dt_ids[] = {
 235         {.compatible = "arm,mali-300"},
 236     /*-------------------------------------------------------*/
 237     /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */
 238         // {.compatible = "arm,mali-400"},
 239         {.compatible = "arm,mali400"},
 240     /*-------------------------------------------------------*/
 241         {.compatible = "arm,mali-450"},
 242         {.compatible = "arm,mali-470"},
 243         {},
 244 };
 245
 246 MODULE_DEVICE_TABLE(of, base_dt_ids);
 247 #endif
 248
 249 /* The Mali device driver struct */
 250 static struct platform_driver mali_platform_driver = {
 251         .probe  = mali_probe,
 252         .remove = mali_remove,
 253 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 254         .pm = &mali_dev_ext_pm_ops,
 255 #endif
 256         .driver =
 257         {
 258                 .name   = MALI_GPU_NAME_UTGARD,
 259                 .owner  = THIS_MODULE,
 260                 .bus = &platform_bus_type,
 261 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
 262                 .pm = &mali_dev_pm_ops,
 263 #endif
 264 #ifdef CONFIG_MALI_DT
 265                 .of_match_table = of_match_ptr(base_dt_ids),
 266 #endif
 267         },
 268 };
 269
 270 /* Linux misc device operations (/dev/mali) */
 271 struct file_operations mali_fops = {
 272         .owner = THIS_MODULE,
 273         .open = mali_open,
 274         .release = mali_release,
 275 #ifdef HAVE_UNLOCKED_IOCTL
 276         .unlocked_ioctl = mali_ioctl,
 277 #else
 278         .ioctl = mali_ioctl,
 279 #endif
 280         .compat_ioctl = mali_ioctl,
 281         .mmap = mali_mmap
 282 };
 283
 284 #if MALI_ENABLE_CPU_CYCLES
 285 void mali_init_cpu_time_counters(int reset, int enable_divide_by_64)
 286 {
 287         /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */
 288         u32 write_value;
 289
 290         /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */
 291         /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */
 292         asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f));
 293
 294
 295         /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */
 296         write_value = 1 << 0; /* Bit 0 set. Enable counters */
 297         if (reset) {
 298                 write_value |= 1 << 1; /* Reset event counters */
 299                 write_value |= 1 << 2; /* Reset cycle counter  */
 300         }
 301         if (enable_divide_by_64) {
 302                 write_value |= 1 << 3; /* Enable the Clock divider by 64 */
 303         }
 304         write_value |= 1 << 4; /* Export enable. Not needed */
 305         asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value));
 306
 307         /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */
 308         asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f));
 309
 310
 311         /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */
 312         /* User mode access to the Performance Monitors enabled. */
 313         /* Lets User space read cpu clock cycles */
 314         asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1));
 315 }
 316
 317 /** A timer function that configures the cycle clock counter on current CPU.
 318  * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this
 319  * function to trigger on all Cpus during module load.
 320  */
 321 static void mali_init_cpu_clock_timer_func(unsigned long data)
 322 {
 323         int reset_counters, enable_divide_clock_counter_by_64;
 324         int current_cpu = raw_smp_processor_id();
 325         unsigned int sample0;
 326         unsigned int sample1;
 327
 328         MALI_IGNORE(data);
 329
 330         reset_counters = 1;
 331         enable_divide_clock_counter_by_64 = 0;
 332         mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64);
 333
 334         sample0 = mali_get_cpu_cyclecount();
 335         sample1 = mali_get_cpu_cyclecount();
 336
 337         MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1));
 338 }
 339
 340 /** A timer functions for storing current time on all cpus.
 341  * Used for checking if the clocks have similar values or if they are drifting.
 342  */
 343 static void mali_print_cpu_clock_timer_func(unsigned long data)
 344 {
 345         int current_cpu = raw_smp_processor_id();
 346         unsigned int sample0;
 347
 348         MALI_IGNORE(data);
 349         sample0 = mali_get_cpu_cyclecount();
 350         if (current_cpu < 8) {
 351                 mali_cpu_clock_last_value[current_cpu] = sample0;
 352         }
 353 }
 354
 355 /** Init the performance registers on all CPUs to count clock cycles.
 356  * For init \a print_only should be 0.
 357  * If \a print_only is 1, it will intead print the current clock value of all CPUs.
 358  */
 359 void mali_init_cpu_time_counters_on_all_cpus(int print_only)
 360 {
 361         int i = 0;
 362         int cpu_number;
 363         int jiffies_trigger;
 364         int jiffies_wait;
 365
 366         jiffies_wait = 2;
 367         jiffies_trigger = jiffies + jiffies_wait;
 368
 369         for (i = 0 ; i < 8 ; i++) {
 370                 init_timer(&mali_init_cpu_clock_timers[i]);
 371                 if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func;
 372                 else            mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func;
 373                 mali_init_cpu_clock_timers[i].expires = jiffies_trigger ;
 374         }
 375         cpu_number = cpumask_first(cpu_online_mask);
 376         for (i = 0 ; i < 8 ; i++) {
 377                 int next_cpu;
 378                 add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number);
 379                 next_cpu = cpumask_next(cpu_number, cpu_online_mask);
 380                 if (next_cpu >= nr_cpu_ids) break;
 381                 cpu_number = next_cpu;
 382         }
 383
 384         while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait);
 385
 386         for (i = 0 ; i < 8 ; i++) {
 387                 del_timer_sync(&mali_init_cpu_clock_timers[i]);
 388         }
 389
 390         if (print_only) {
 391                 if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) {
 392                         /* Diff can be printed if we want to check if the clocks are in sync
 393                         int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/
 394                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1]));
 395                 } else {
 396                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3]));
 397                 }
 398         }
 399 }
 400 #endif
 401
 402 int mali_module_init(void)
 403 {
 404         int err = 0;
 405
 406         MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION));
 407         MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__));
 408         MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING));
 409
 410         I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.",
 411                 SVN_REV_STRING,
 412                 RK_KO_VER,
 413                 __TIME__,
 414                 __DATE__);
 415
 416 #if MALI_ENABLE_CPU_CYCLES
 417         mali_init_cpu_time_counters_on_all_cpus(0);
 418         MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n"));
 419         /* Printing the current cpu counters */
 420         mali_init_cpu_time_counters_on_all_cpus(1);
 421 #endif
 422
 423         /* Initialize module wide settings */
 424 #ifdef MALI_FAKE_PLATFORM_DEVICE
 425 #ifndef CONFIG_MALI_DT
 426         MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n"));
 427         err = mali_platform_device_register();
 428         if (0 != err) {
 429                 return err;
 430         }
 431 #endif
 432 #endif
 433
 434         MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n"));
 435
 436         err = platform_driver_register(&mali_platform_driver);
 437
 438         if (0 != err) {
 439                 MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err));
 440 #ifdef MALI_FAKE_PLATFORM_DEVICE
 441 #ifndef CONFIG_MALI_DT
 442                 mali_platform_device_unregister();
 443 #endif
 444 #endif
 445                 mali_platform_device = NULL;
 446                 return err;
 447         }
 448
 449 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 450         err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
 451         if (0 != err) {
 452                 /* No biggie if we wheren't able to initialize the profiling */
 453                 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
 454         }
 455 #endif
 456
 457         /* Tracing the current frequency and voltage from boot/insmod*/
 458 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 459         /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/
 460         mali_get_current_gpu_clk_item(&mali_gpu_clk[0]);
 461         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 462                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 463                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 464                                       mali_gpu_clk[0].clock,
 465                                       mali_gpu_clk[0].vol / 1000,
 466                                       0, 0, 0);
 467 #endif
 468
 469         MALI_PRINT(("Mali device driver loaded\n"));
 470
 471         return 0; /* Success */
 472 }
 473
 474 void mali_module_exit(void)
 475 {
 476         MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION));
 477
 478         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n"));
 479
 480         platform_driver_unregister(&mali_platform_driver);
 481
 482 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 483 #ifndef CONFIG_MALI_DT
 484         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n"));
 485         mali_platform_device_unregister();
 486 #endif
 487 #endif
 488
 489         /* Tracing the current frequency and voltage from rmmod*/
 490         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 491                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 492                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 493                                       0,
 494                                       0,
 495                                       0, 0, 0);
 496
 497 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 498         _mali_internal_profiling_term();
 499 #endif
 500
 501         MALI_PRINT(("Mali device driver unloaded\n"));
 502 }
 503
 504 #ifdef CONFIG_MALI_DEVFREQ
 505 struct mali_device *mali_device_alloc(void)
 506 {
 507         return kzalloc(sizeof(struct mali_device), GFP_KERNEL);
 508 }
 509
 510 void mali_device_free(struct mali_device *mdev)
 511 {
 512         kfree(mdev);
 513 }
 514 #endif
 515
 516 static int mali_probe(struct platform_device *pdev)
 517 {
 518         int err;
 519 #ifdef CONFIG_MALI_DEVFREQ
 520         struct mali_device *mdev;
 521 #endif
 522
 523         MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name));
 524
 525         if (NULL != mali_platform_device) {
 526                 /* Already connected to a device, return error */
 527                 MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device."));
 528                 return -EEXIST;
 529         }
 530
 531         mali_platform_device = pdev;
 532
 533         dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n",
 534                 mali_platform_device->num_resources);
 535
 536         {
 537                 int i = 0;
 538
 539                 for(i = 0; i < mali_platform_device->num_resources; i++)
 540                         dev_info(&pdev->dev,
 541                                  "resource[%d].start = 0x%pa\n",
 542                                  i,
 543                                  &mali_platform_device->resource[i].start);
 544         }
 545
 546 #ifdef CONFIG_MALI_DT
 547         /* If we use DT to initialize our DDK, we have to prepare somethings. */
 548         err = mali_platform_device_init(mali_platform_device);
 549         if (0 != err) {
 550                 MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device."));
 551                 mali_platform_device = NULL;
 552                 return -EFAULT;
 553         }
 554 #endif
 555
 556 #ifdef CONFIG_MALI_DEVFREQ
 557         mdev = mali_device_alloc();
 558         if (!mdev) {
 559                 MALI_PRINT_ERROR(("Can't allocate mali device private data\n"));
 560                 return -ENOMEM;
 561         }
 562
 563         mdev->dev = &pdev->dev;
 564         dev_set_drvdata(mdev->dev, mdev);
 565
 566         /*Initilization clock and regulator*/
 567 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
 568                         && defined(CONFIG_REGULATOR)
 569         mdev->regulator = regulator_get_optional(mdev->dev, "mali");
 570         if (IS_ERR_OR_NULL(mdev->regulator)) {
 571                 MALI_DEBUG_PRINT(2, ("Continuing without Mali regulator control\n"));
 572                 mdev->regulator = NULL;
 573                 /* Allow probe to continue without regulator */
 574         }
 575 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
 576
 577 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) && defined(CONFIG_OF) \
 578                         && defined(CONFIG_PM_OPP)
 579         /* Register the OPPs if they are available in device tree */
 580         if (dev_pm_opp_of_add_table(mdev->dev) < 0)
 581                 MALI_DEBUG_PRINT(3, ("OPP table not found\n"));
 582 #endif
 583
 584         /* Need to name the gpu clock "clk_mali" in the device tree */
 585         mdev->clock = clk_get(mdev->dev, "clk_mali");
 586         if (IS_ERR_OR_NULL(mdev->clock)) {
 587                 MALI_DEBUG_PRINT(2, ("Continuing without Mali clock control\n"));
 588                 mdev->clock = NULL;
 589                 /* Allow probe to continue without clock. */
 590         } else {
 591                 err = clk_prepare_enable(mdev->clock);
 592                 if (err) {
 593                         MALI_PRINT_ERROR(("Failed to prepare and enable clock (%d)\n", err));
 594                         goto clock_prepare_failed;
 595                 }
 596         }
 597
 598         /* initilize pm metrics related */
 599         if (mali_pm_metrics_init(mdev) < 0) {
 600                 MALI_DEBUG_PRINT(2, ("mali pm metrics init failed\n"));
 601                 goto pm_metrics_init_failed;
 602         }
 603
 604         if (mali_devfreq_init(mdev) < 0) {
 605                 MALI_DEBUG_PRINT(2, ("mali devfreq init failed\n"));
 606                 goto devfreq_init_failed;
 607         }
 608 #endif
 609
 610
 611         if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) {
 612                 /* Initialize the Mali GPU HW specified by pdev */
 613                 if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) {
 614                         /* Register a misc device (so we are accessible from user space) */
 615                         err = mali_miscdevice_register(pdev);
 616                         if (0 == err) {
 617                                 /* Setup sysfs entries */
 618                                 err = mali_sysfs_register(mali_dev_name);
 619
 620                                 if (0 == err) {
 621                                         MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name));
 622
 623                                         return 0;
 624                                 } else {
 625                                         MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries"));
 626                                 }
 627                                 mali_miscdevice_unregister();
 628                         } else {
 629                                 MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device."));
 630                         }
 631                         mali_terminate_subsystems();
 632                 } else {
 633                         MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver."));
 634                 }
 635                 _mali_osk_wq_term();
 636         }
 637
 638 #ifdef CONFIG_MALI_DEVFREQ
 639         mali_devfreq_term(mdev);
 640 devfreq_init_failed:
 641         mali_pm_metrics_term(mdev);
 642 pm_metrics_init_failed:
 643         clk_disable_unprepare(mdev->clock);
 644 clock_prepare_failed:
 645         clk_put(mdev->clock);
 646 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \
 647                         && defined(CONFIG_PM_OPP)
 648         dev_pm_opp_of_remove_table(mdev->dev);
 649 #endif
 650
 651 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
 652                         && defined(CONFIG_REGULATOR)
 653         regulator_put(mdev->regulator);
 654 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
 655         mali_device_free(mdev);
 656 #endif
 657
 658 #ifdef CONFIG_MALI_DT
 659         mali_platform_device_deinit(mali_platform_device);
 660 #endif
 661         mali_platform_device = NULL;
 662         return -EFAULT;
 663 }
 664
 665 static int mali_remove(struct platform_device *pdev)
 666 {
 667 #ifdef CONFIG_MALI_DEVFREQ
 668         struct mali_device *mdev = dev_get_drvdata(&pdev->dev);
 669 #endif
 670
 671         MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name));
 672         mali_sysfs_unregister();
 673         mali_miscdevice_unregister();
 674         mali_terminate_subsystems();
 675         _mali_osk_wq_term();
 676
 677 #ifdef CONFIG_MALI_DEVFREQ
 678         mali_devfreq_term(mdev);
 679
 680         mali_pm_metrics_term(mdev);
 681
 682         if (mdev->clock) {
 683                 clk_disable_unprepare(mdev->clock);
 684                 clk_put(mdev->clock);
 685                 mdev->clock = NULL;
 686         }
 687 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \
 688                         && defined(CONFIG_PM_OPP)
 689         dev_pm_opp_of_remove_table(mdev->dev);
 690 #endif
 691
 692 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
 693                         && defined(CONFIG_REGULATOR)
 694         regulator_put(mdev->regulator);
 695 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
 696         mali_device_free(mdev);
 697 #endif
 698
 699 #ifdef CONFIG_MALI_DT
 700         mali_platform_device_deinit(mali_platform_device);
 701 #endif
 702         mali_platform_device = NULL;
 703         return 0;
 704 }
 705
 706 static int mali_miscdevice_register(struct platform_device *pdev)
 707 {
 708         int err;
 709
 710         mali_miscdevice.minor = MISC_DYNAMIC_MINOR;
 711         mali_miscdevice.name = mali_dev_name;
 712         mali_miscdevice.fops = &mali_fops;
 713         mali_miscdevice.parent = get_device(&pdev->dev);
 714
 715         err = misc_register(&mali_miscdevice);
 716         if (0 != err) {
 717                 MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err));
 718         }
 719
 720         return err;
 721 }
 722
 723 static void mali_miscdevice_unregister(void)
 724 {
 725         misc_deregister(&mali_miscdevice);
 726 }
 727
 728 static int mali_driver_suspend_scheduler(struct device *dev)
 729 {
 730 #ifdef CONFIG_MALI_DEVFREQ
 731         struct mali_device *mdev = dev_get_drvdata(dev);
 732         if (!mdev)
 733                 return -ENODEV;
 734 #endif
 735
 736 #if defined(CONFIG_MALI_DEVFREQ) && \
 737                 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
 738         devfreq_suspend_device(mdev->devfreq);
 739 #endif
 740
 741         mali_pm_os_suspend(MALI_TRUE);
 742         /* Tracing the frequency and voltage after mali is suspended */
 743         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 744                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 745                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 746                                       0,
 747                                       0,
 748                                       0, 0, 0);
 749         return 0;
 750 }
 751
 752 static int mali_driver_resume_scheduler(struct device *dev)
 753 {
 754 #ifdef CONFIG_MALI_DEVFREQ
 755         struct mali_device *mdev = dev_get_drvdata(dev);
 756         if (!mdev)
 757                 return -ENODEV;
 758 #endif
 759
 760         /* Tracing the frequency and voltage after mali is resumed */
 761 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 762         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 763         if (is_first_resume == 1) {
 764                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 765                 is_first_resume = 0;
 766         }
 767         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 768                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 769                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 770                                       mali_gpu_clk[1].clock,
 771                                       mali_gpu_clk[1].vol / 1000,
 772                                       0, 0, 0);
 773 #endif
 774         mali_pm_os_resume();
 775
 776 #if defined(CONFIG_MALI_DEVFREQ) && \
 777                 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
 778         devfreq_resume_device(mdev->devfreq);
 779 #endif
 780
 781         return 0;
 782 }
 783
 784 #ifdef CONFIG_PM_RUNTIME
 785 static int mali_driver_runtime_suspend(struct device *dev)
 786 {
 787 #ifdef CONFIG_MALI_DEVFREQ
 788         struct mali_device *mdev = dev_get_drvdata(dev);
 789         if (!mdev)
 790                 return -ENODEV;
 791 #endif
 792
 793         if (MALI_TRUE == mali_pm_runtime_suspend()) {
 794                 /* Tracing the frequency and voltage after mali is suspended */
 795                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 796                                               MALI_PROFILING_EVENT_CHANNEL_GPU |
 797                                               MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 798                                               0,
 799                                               0,
 800                                               0, 0, 0);
 801
 802 #if defined(CONFIG_MALI_DEVFREQ) && \
 803                 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
 804                 MALI_DEBUG_PRINT(4, ("devfreq_suspend_device: stop devfreq monitor\n"));
 805                 devfreq_suspend_device(mdev->devfreq);
 806 #endif
 807
 808                 return 0;
 809         } else {
 810                 return -EBUSY;
 811         }
 812 }
 813
 814 static int mali_driver_runtime_resume(struct device *dev)
 815 {
 816 #ifdef CONFIG_MALI_DEVFREQ
 817         struct mali_device *mdev = dev_get_drvdata(dev);
 818         if (!mdev)
 819                 return -ENODEV;
 820 #endif
 821
 822         /* Tracing the frequency and voltage after mali is resumed */
 823 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 824         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 825         if (is_first_resume == 1) {
 826                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 827                 is_first_resume = 0;
 828         }
 829         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 830                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 831                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 832                                       mali_gpu_clk[1].clock,
 833                                       mali_gpu_clk[1].vol / 1000,
 834                                       0, 0, 0);
 835 #endif
 836
 837         mali_pm_runtime_resume();
 838
 839 #if defined(CONFIG_MALI_DEVFREQ) && \
 840                 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
 841         MALI_DEBUG_PRINT(4, ("devfreq_resume_device: start devfreq monitor\n"));
 842         devfreq_resume_device(mdev->devfreq);
 843 #endif
 844         return 0;
 845 }
 846
 847 static int mali_driver_runtime_idle(struct device *dev)
 848 {
 849         /* Nothing to do */
 850         return 0;
 851 }
 852 #endif
 853
 854 static int mali_open(struct inode *inode, struct file *filp)
 855 {
 856         struct mali_session_data *session_data;
 857         _mali_osk_errcode_t err;
 858
 859         /* input validation */
 860         if (mali_miscdevice.minor != iminor(inode)) {
 861                 MALI_PRINT_ERROR(("mali_open() Minor does not match\n"));
 862                 return -ENODEV;
 863         }
 864
 865         /* allocated struct to track this session */
 866         err = _mali_ukk_open((void **)&session_data);
 867         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 868
 869         /* initialize file pointer */
 870         filp->f_pos = 0;
 871
 872         /* link in our session data */
 873         filp->private_data = (void *)session_data;
 874
 875         filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping;
 876
 877         return 0;
 878 }
 879
 880 static int mali_release(struct inode *inode, struct file *filp)
 881 {
 882         _mali_osk_errcode_t err;
 883
 884         /* input validation */
 885         if (mali_miscdevice.minor != iminor(inode)) {
 886                 MALI_PRINT_ERROR(("mali_release() Minor does not match\n"));
 887                 return -ENODEV;
 888         }
 889
 890         err = _mali_ukk_close((void **)&filp->private_data);
 891         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 892
 893         return 0;
 894 }
 895
 896 int map_errcode(_mali_osk_errcode_t err)
 897 {
 898         switch (err) {
 899         case _MALI_OSK_ERR_OK :
 900                 return 0;
 901         case _MALI_OSK_ERR_FAULT:
 902                 return -EFAULT;
 903         case _MALI_OSK_ERR_INVALID_FUNC:
 904                 return -ENOTTY;
 905         case _MALI_OSK_ERR_INVALID_ARGS:
 906                 return -EINVAL;
 907         case _MALI_OSK_ERR_NOMEM:
 908                 return -ENOMEM;
 909         case _MALI_OSK_ERR_TIMEOUT:
 910                 return -ETIMEDOUT;
 911         case _MALI_OSK_ERR_RESTARTSYSCALL:
 912                 return -ERESTARTSYS;
 913         case _MALI_OSK_ERR_ITEM_NOT_FOUND:
 914                 return -ENOENT;
 915         default:
 916                 return -EFAULT;
 917         }
 918 }
 919
 920 #ifdef HAVE_UNLOCKED_IOCTL
 921 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 922 #else
 923 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
 924 #endif
 925 {
 926         int err;
 927         struct mali_session_data *session_data;
 928
 929 #ifndef HAVE_UNLOCKED_IOCTL
 930         /* inode not used */
 931         (void)inode;
 932 #endif
 933
 934         MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg));
 935
 936         session_data = (struct mali_session_data *)filp->private_data;
 937         if (NULL == session_data) {
 938                 MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n"));
 939                 return -ENOTTY;
 940         }
 941
 942         if (NULL == (void *)arg) {
 943                 MALI_DEBUG_PRINT(7, ("arg was NULL\n"));
 944                 return -ENOTTY;
 945         }
 946
 947         switch (cmd) {
 948         case MALI_IOC_WAIT_FOR_NOTIFICATION:
 949                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64)));
 950                 err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg);
 951                 break;
 952
 953         case MALI_IOC_GET_API_VERSION_V2:
 954                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64)));
 955                 err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg);
 956                 break;
 957
 958         case MALI_IOC_GET_API_VERSION:
 959                 err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg);
 960                 break;
 961
 962         case MALI_IOC_POST_NOTIFICATION:
 963                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64)));
 964                 err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg);
 965                 break;
 966
 967     /* rk_ext : 从对 r5p0-01rel0 集成开始, 不再使用. */
 968 #if 0
 969         case MALI_IOC_GET_MALI_VERSION_IN_RK30:
 970                 err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg);
 971                 break;
 972 #else
 973     case MALI_IOC_GET_RK_KO_VERSION:
 974                 err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg);
 975                 break;
 976 #endif
 977
 978         case MALI_IOC_GET_USER_SETTINGS:
 979                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64)));
 980                 err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg);
 981                 break;
 982
 983         case MALI_IOC_REQUEST_HIGH_PRIORITY:
 984                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64)));
 985                 err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg);
 986                 break;
 987
 988         case MALI_IOC_PENDING_SUBMIT:
 989                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64)));
 990                 err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg);
 991                 break;
 992
 993 #if defined(CONFIG_MALI400_PROFILING)
 994         case MALI_IOC_PROFILING_ADD_EVENT:
 995                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64)));
 996                 err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg);
 997                 break;
 998
 999         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS:
1000                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64)));
1001                 err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg);
1002                 break;
1003
1004         case MALI_IOC_PROFILING_STREAM_FD_GET:
1005                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64)));
1006                 err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg);
1007                 break;
1008
1009         case MALI_IOC_PROILING_CONTROL_SET:
1010                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64)));
1011                 err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg);
1012                 break;
1013 #else
1014
1015         case MALI_IOC_PROFILING_ADD_EVENT:          /* FALL-THROUGH */
1016         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */
1017                 MALI_DEBUG_PRINT(2, ("Profiling not supported\n"));
1018                 err = -ENOTTY;
1019                 break;
1020 #endif
1021
1022         case MALI_IOC_PROFILING_MEMORY_USAGE_GET:
1023                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64)));
1024                 err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg);
1025                 break;
1026
1027         case MALI_IOC_MEM_ALLOC:
1028                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64)));
1029                 err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg);
1030                 break;
1031
1032         case MALI_IOC_MEM_FREE:
1033                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64)));
1034                 err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg);
1035                 break;
1036
1037         case MALI_IOC_MEM_BIND:
1038                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64)));
1039                 err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg);
1040                 break;
1041
1042         case MALI_IOC_MEM_UNBIND:
1043                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64)));
1044                 err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg);
1045                 break;
1046
1047         case MALI_IOC_MEM_COW:
1048                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64)));
1049                 err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg);
1050                 break;
1051
1052         case MALI_IOC_MEM_COW_MODIFY_RANGE:
1053                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64)));
1054                 err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg);
1055                 break;
1056
1057         case MALI_IOC_MEM_RESIZE:
1058                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64)));
1059                 err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg);
1060                 break;
1061
1062         case MALI_IOC_MEM_WRITE_SAFE:
1063                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64)));
1064                 err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg);
1065                 break;
1066
1067         case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE:
1068                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64)));
1069                 err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg);
1070                 break;
1071
1072         case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE:
1073                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64)));
1074                 err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg);
1075                 break;
1076
1077         case MALI_IOC_MEM_DMA_BUF_GET_SIZE:
1078 #ifdef CONFIG_DMA_SHARED_BUFFER
1079                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64)));
1080                 err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg);
1081 #else
1082                 MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n"));
1083                 err = -ENOTTY;
1084 #endif
1085                 break;
1086
1087         case MALI_IOC_PP_START_JOB:
1088                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64)));
1089                 err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg);
1090                 break;
1091
1092         case MALI_IOC_PP_AND_GP_START_JOB:
1093                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64)));
1094                 err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg);
1095                 break;
1096
1097         case MALI_IOC_PP_NUMBER_OF_CORES_GET:
1098                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64)));
1099                 err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg);
1100                 break;
1101
1102         case MALI_IOC_PP_CORE_VERSION_GET:
1103                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64)));
1104                 err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg);
1105                 break;
1106
1107         case MALI_IOC_PP_DISABLE_WB:
1108                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64)));
1109                 err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg);
1110                 break;
1111
1112         case MALI_IOC_GP2_START_JOB:
1113                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64)));
1114                 err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg);
1115                 break;
1116
1117         case MALI_IOC_GP2_NUMBER_OF_CORES_GET:
1118                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64)));
1119                 err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg);
1120                 break;
1121
1122         case MALI_IOC_GP2_CORE_VERSION_GET:
1123                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64)));
1124                 err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg);
1125                 break;
1126
1127         case MALI_IOC_GP2_SUSPEND_RESPONSE:
1128                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64)));
1129                 err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg);
1130                 break;
1131
1132         case MALI_IOC_VSYNC_EVENT_REPORT:
1133                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64)));
1134                 err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg);
1135                 break;
1136
1137         case MALI_IOC_TIMELINE_GET_LATEST_POINT:
1138                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64)));
1139                 err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg);
1140                 break;
1141         case MALI_IOC_TIMELINE_WAIT:
1142                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64)));
1143                 err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg);
1144                 break;
1145         case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE:
1146                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64)));
1147                 err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg);
1148                 break;
1149         case MALI_IOC_SOFT_JOB_START:
1150                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64)));
1151                 err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg);
1152                 break;
1153         case MALI_IOC_SOFT_JOB_SIGNAL:
1154                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64)));
1155                 err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg);
1156                 break;
1157
1158         default:
1159                 MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg));
1160                 err = -ENOTTY;
1161         };
1162
1163         return err;
1164 }
1165
1166 late_initcall_sync(mali_module_init);
1167 module_exit(mali_module_exit);
1168
1169 MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE);
1170 MODULE_AUTHOR("ARM Ltd.");
1171 MODULE_VERSION(SVN_REV_STRING);