drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c

   1 /**
   2  * Copyright (C) 2010-2014 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11
  12 /**
  13  * @file mali_kernel_linux.c
  14  * Implementation of the Linux device driver entrypoints
  15  */
  16 #include "../platform/rk30/custom_log.h"
  17 #include "../platform/rk30/rk_ext.h"
  18
  19 #include <linux/module.h>   /* kernel module definitions */
  20 #include <linux/fs.h>       /* file system operations */
  21 #include <linux/cdev.h>     /* character device definitions */
  22 #include <linux/mm.h>       /* memory manager definitions */
  23 #include <linux/mali/mali_utgard_ioctl.h>
  24 #include <linux/version.h>
  25 #include <linux/device.h>
  26 #include "mali_kernel_license.h"
  27 #include <linux/platform_device.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/bug.h>
  30 #include <linux/of.h>
  31
  32 #include <linux/mali/mali_utgard.h>
  33 #include "mali_kernel_common.h"
  34 #include "mali_session.h"
  35 #include "mali_kernel_core.h"
  36 #include "mali_osk.h"
  37 #include "mali_kernel_linux.h"
  38 #include "mali_ukk.h"
  39 #include "mali_ukk_wrappers.h"
  40 #include "mali_kernel_sysfs.h"
  41 #include "mali_pm.h"
  42 #include "mali_kernel_license.h"
  43 #include "mali_memory.h"
  44 #include "mali_memory_dma_buf.h"
  45 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
  46 #include "mali_profiling_internal.h"
  47 #endif
  48 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
  49 #include "mali_osk_profiling.h"
  50 #include "mali_dvfs_policy.h"
  51 static int is_first_resume = 1;
  52 /*Store the clk and vol for boot/insmod and mali_resume*/
  53 static struct mali_gpu_clk_item mali_gpu_clk[2];
  54 #endif
  55
  56 /* Streamline support for the Mali driver */
  57 #if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING)
  58 /* Ask Linux to create the tracepoints */
  59 #define CREATE_TRACE_POINTS
  60 #include "mali_linux_trace.h"
  61
  62 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event);
  63 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter);
  64 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters);
  65 #endif /* CONFIG_TRACEPOINTS */
  66
  67 /* from the __malidrv_build_info.c file that is generated during build */
  68 extern const char *__malidrv_build_info(void);
  69
  70 /* Module parameter to control log level */
  71 int mali_debug_level = 2;
  72 module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
  73 MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output");
  74
  75 extern int mali_max_job_runtime;
  76 module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
  77 MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what");
  78
  79 extern int mali_l2_max_reads;
  80 module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH);
  81 MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache");
  82
  83 extern unsigned int mali_dedicated_mem_start;
  84 module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH);
  85 MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory.");
  86
  87 extern unsigned int mali_dedicated_mem_size;
  88 module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
  89 MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory.");
  90
  91 extern unsigned int mali_shared_mem_size;
  92 module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
  93 MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory.");
  94
  95 #if defined(CONFIG_MALI400_PROFILING)
  96 extern int mali_boot_profiling;
  97 module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH);
  98 MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization");
  99 #endif
 100
 101 extern int mali_max_pp_cores_group_1;
 102 module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH);
 103 MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group.");
 104
 105 extern int mali_max_pp_cores_group_2;
 106 module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH);
 107 MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only).");
 108
 109 #if defined(CONFIG_MALI_DVFS)
 110 /** the max fps the same as display vsync default 60, can set by module insert parameter */
 111 extern int mali_max_system_fps;
 112 module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 113 MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC.");
 114
 115 /** a lower limit on their desired FPS default 58, can set by module insert parameter*/
 116 extern int mali_desired_fps;
 117 module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 118 MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps");
 119 #endif
 120
 121 #if MALI_ENABLE_CPU_CYCLES
 122 #include <linux/cpumask.h>
 123 #include <linux/timer.h>
 124 #include <asm/smp.h>
 125 static struct timer_list mali_init_cpu_clock_timers[8];
 126 static u32 mali_cpu_clock_last_value[8] = {0,};
 127 #endif
 128
 129 /* Export symbols from common code: mali_user_settings.c */
 130 #include "mali_user_settings_db.h"
 131 EXPORT_SYMBOL(mali_set_user_setting);
 132 EXPORT_SYMBOL(mali_get_user_setting);
 133
 134 static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */
 135
 136 /* This driver only supports one Mali device, and this variable stores this single platform device */
 137 struct platform_device *mali_platform_device = NULL;
 138
 139 /* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */
 140 static struct miscdevice mali_miscdevice = { 0, };
 141
 142 static int mali_miscdevice_register(struct platform_device *pdev);
 143 static void mali_miscdevice_unregister(void);
 144
 145 static int mali_open(struct inode *inode, struct file *filp);
 146 static int mali_release(struct inode *inode, struct file *filp);
 147 #ifdef HAVE_UNLOCKED_IOCTL
 148 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 149 #else
 150 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
 151 #endif
 152
 153 static int mali_probe(struct platform_device *pdev);
 154 static int mali_remove(struct platform_device *pdev);
 155
 156 static int mali_driver_suspend_scheduler(struct device *dev);
 157 static int mali_driver_resume_scheduler(struct device *dev);
 158
 159 #ifdef CONFIG_PM_RUNTIME
 160 static int mali_driver_runtime_suspend(struct device *dev);
 161 static int mali_driver_runtime_resume(struct device *dev);
 162 static int mali_driver_runtime_idle(struct device *dev);
 163 #endif
 164
 165 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 166 #if defined(CONFIG_MALI_DT)
 167 extern int mali_platform_device_init(struct platform_device *device);
 168 extern int mali_platform_device_deinit(struct platform_device *device);
 169 #else
 170 extern int mali_platform_device_register(void);
 171 extern int mali_platform_device_unregister(void);
 172 #endif
 173 #endif
 174
 175 /* Linux power management operations provided by the Mali device driver */
 176 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 177 struct pm_ext_ops mali_dev_ext_pm_ops = {
 178         .base =
 179         {
 180                 .suspend = mali_driver_suspend_scheduler,
 181                 .resume = mali_driver_resume_scheduler,
 182                 .freeze = mali_driver_suspend_scheduler,
 183                 .thaw =   mali_driver_resume_scheduler,
 184         },
 185 };
 186 #else
 187 static const struct dev_pm_ops mali_dev_pm_ops = {
 188 #ifdef CONFIG_PM_RUNTIME
 189         .runtime_suspend = mali_driver_runtime_suspend,
 190         .runtime_resume = mali_driver_runtime_resume,
 191         .runtime_idle = mali_driver_runtime_idle,
 192 #endif
 193         .suspend = mali_driver_suspend_scheduler,
 194         .resume = mali_driver_resume_scheduler,
 195         .freeze = mali_driver_suspend_scheduler,
 196         .thaw = mali_driver_resume_scheduler,
 197         .poweroff = mali_driver_suspend_scheduler,
 198 };
 199 #endif
 200
 201 #ifdef CONFIG_MALI_DT
 202 static struct of_device_id base_dt_ids[] = {
 203         {.compatible = "arm,mali-300"},
 204     /*-------------------------------------------------------*/
 205     /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */
 206         // {.compatible = "arm,mali-400"},
 207         {.compatible = "arm,mali400"},
 208     /*-------------------------------------------------------*/
 209         {.compatible = "arm,mali-450"},
 210         {.compatible = "arm,mali-utgard"},
 211         {},
 212 };
 213
 214 MODULE_DEVICE_TABLE(of, base_dt_ids);
 215 #endif
 216
 217 /* The Mali device driver struct */
 218 static struct platform_driver mali_platform_driver = {
 219         .probe  = mali_probe,
 220         .remove = mali_remove,
 221 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 222         .pm = &mali_dev_ext_pm_ops,
 223 #endif
 224         .driver =
 225         {
 226                 .name   = MALI_GPU_NAME_UTGARD,
 227                 .owner  = THIS_MODULE,
 228                 .bus = &platform_bus_type,
 229 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
 230                 .pm = &mali_dev_pm_ops,
 231 #endif
 232 #ifdef CONFIG_MALI_DT
 233                 .of_match_table = of_match_ptr(base_dt_ids),
 234 #endif
 235         },
 236 };
 237
 238 /* Linux misc device operations (/dev/mali) */
 239 struct file_operations mali_fops = {
 240         .owner = THIS_MODULE,
 241         .open = mali_open,
 242         .release = mali_release,
 243 #ifdef HAVE_UNLOCKED_IOCTL
 244         .unlocked_ioctl = mali_ioctl,
 245 #else
 246         .ioctl = mali_ioctl,
 247 #endif
 248         .compat_ioctl = mali_ioctl,
 249         .mmap = mali_mmap
 250 };
 251
 252 #if MALI_ENABLE_CPU_CYCLES
 253 void mali_init_cpu_time_counters(int reset, int enable_divide_by_64)
 254 {
 255         /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */
 256         u32 write_value;
 257
 258         /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */
 259         /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */
 260         asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f));
 261
 262
 263         /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */
 264         write_value = 1 << 0; /* Bit 0 set. Enable counters */
 265         if (reset) {
 266                 write_value |= 1 << 1; /* Reset event counters */
 267                 write_value |= 1 << 2; /* Reset cycle counter  */
 268         }
 269         if (enable_divide_by_64) {
 270                 write_value |= 1 << 3; /* Enable the Clock divider by 64 */
 271         }
 272         write_value |= 1 << 4; /* Export enable. Not needed */
 273         asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value));
 274
 275         /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */
 276         asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f));
 277
 278
 279         /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */
 280         /* User mode access to the Performance Monitors enabled. */
 281         /* Lets User space read cpu clock cycles */
 282         asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1));
 283 }
 284
 285 /** A timer function that configures the cycle clock counter on current CPU.
 286  * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this
 287  * function to trigger on all Cpus during module load.
 288  */
 289 static void mali_init_cpu_clock_timer_func(unsigned long data)
 290 {
 291         int reset_counters, enable_divide_clock_counter_by_64;
 292         int current_cpu = raw_smp_processor_id();
 293         unsigned int sample0;
 294         unsigned int sample1;
 295
 296         MALI_IGNORE(data);
 297
 298         reset_counters = 1;
 299         enable_divide_clock_counter_by_64 = 0;
 300         mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64);
 301
 302         sample0 = mali_get_cpu_cyclecount();
 303         sample1 = mali_get_cpu_cyclecount();
 304
 305         MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1));
 306 }
 307
 308 /** A timer functions for storing current time on all cpus.
 309  * Used for checking if the clocks have similar values or if they are drifting.
 310  */
 311 static void mali_print_cpu_clock_timer_func(unsigned long data)
 312 {
 313         int current_cpu = raw_smp_processor_id();
 314         unsigned int sample0;
 315
 316         MALI_IGNORE(data);
 317         sample0 = mali_get_cpu_cyclecount();
 318         if (current_cpu < 8) {
 319                 mali_cpu_clock_last_value[current_cpu] = sample0;
 320         }
 321 }
 322
 323 /** Init the performance registers on all CPUs to count clock cycles.
 324  * For init \a print_only should be 0.
 325  * If \a print_only is 1, it will intead print the current clock value of all CPUs.
 326  */
 327 void mali_init_cpu_time_counters_on_all_cpus(int print_only)
 328 {
 329         int i = 0;
 330         int cpu_number;
 331         int jiffies_trigger;
 332         int jiffies_wait;
 333
 334         jiffies_wait = 2;
 335         jiffies_trigger = jiffies + jiffies_wait;
 336
 337         for (i = 0 ; i < 8 ; i++) {
 338                 init_timer(&mali_init_cpu_clock_timers[i]);
 339                 if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func;
 340                 else            mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func;
 341                 mali_init_cpu_clock_timers[i].expires = jiffies_trigger ;
 342         }
 343         cpu_number = cpumask_first(cpu_online_mask);
 344         for (i = 0 ; i < 8 ; i++) {
 345                 int next_cpu;
 346                 add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number);
 347                 next_cpu = cpumask_next(cpu_number, cpu_online_mask);
 348                 if (next_cpu >= nr_cpu_ids) break;
 349                 cpu_number = next_cpu;
 350         }
 351
 352         while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait);
 353
 354         for (i = 0 ; i < 8 ; i++) {
 355                 del_timer_sync(&mali_init_cpu_clock_timers[i]);
 356         }
 357
 358         if (print_only) {
 359                 if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) {
 360                         /* Diff can be printed if we want to check if the clocks are in sync
 361                         int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/
 362                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1]));
 363                 } else {
 364                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3]));
 365                 }
 366         }
 367 }
 368 #endif
 369
 370 int mali_module_init(void)
 371 {
 372         int err = 0;
 373
 374         MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION));
 375         MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__));
 376         MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING));
 377
 378         I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.",
 379                 SVN_REV_STRING,
 380                 RK_KO_VER,
 381                 __TIME__,
 382                 __DATE__);
 383
 384 #if MALI_ENABLE_CPU_CYCLES
 385         mali_init_cpu_time_counters_on_all_cpus(0);
 386         MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n"));
 387         /* Printing the current cpu counters */
 388         mali_init_cpu_time_counters_on_all_cpus(1);
 389 #endif
 390
 391         /* Initialize module wide settings */
 392 #ifdef MALI_FAKE_PLATFORM_DEVICE
 393 #ifndef CONFIG_MALI_DT
 394         MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n"));
 395         err = mali_platform_device_register();
 396         if (0 != err) {
 397                 return err;
 398         }
 399 #endif
 400 #endif
 401
 402         MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n"));
 403
 404         err = platform_driver_register(&mali_platform_driver);
 405
 406         if (0 != err) {
 407                 MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err));
 408 #ifdef MALI_FAKE_PLATFORM_DEVICE
 409 #ifndef CONFIG_MALI_DT
 410         mali_platform_device_unregister();
 411 #endif
 412 #endif
 413                 mali_platform_device = NULL;
 414                 return err;
 415         }
 416
 417 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 418         err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
 419         if (0 != err) {
 420                 /* No biggie if we wheren't able to initialize the profiling */
 421                 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
 422         }
 423 #endif
 424
 425         /* Tracing the current frequency and voltage from boot/insmod*/
 426 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 427         /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/
 428         mali_get_current_gpu_clk_item(&mali_gpu_clk[0]);
 429         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 430                                           MALI_PROFILING_EVENT_CHANNEL_GPU |
 431                                           MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 432                                           mali_gpu_clk[0].clock,
 433                                           mali_gpu_clk[0].vol / 1000,
 434                                           0, 0, 0);
 435 #endif
 436
 437         MALI_PRINT(("Mali device driver loaded\n"));
 438
 439         return 0; /* Success */
 440 }
 441
 442 void mali_module_exit(void)
 443 {
 444         MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION));
 445
 446         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n"));
 447
 448         platform_driver_unregister(&mali_platform_driver);
 449
 450 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 451 #ifndef CONFIG_MALI_DT
 452         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n"));
 453         mali_platform_device_unregister();
 454 #endif
 455 #endif
 456
 457         /* Tracing the current frequency and voltage from rmmod*/
 458         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 459                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 460                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 461                                       0,
 462                                       0,
 463                                       0, 0, 0);
 464
 465 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 466         _mali_internal_profiling_term();
 467 #endif
 468
 469         MALI_PRINT(("Mali device driver unloaded\n"));
 470 }
 471
 472 static int mali_probe(struct platform_device *pdev)
 473 {
 474         int err;
 475
 476         MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name));
 477
 478         if (NULL != mali_platform_device) {
 479                 /* Already connected to a device, return error */
 480                 MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device."));
 481                 return -EEXIST;
 482         }
 483
 484         mali_platform_device = pdev;
 485
 486         dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n",
 487                 mali_platform_device->num_resources);
 488
 489         {
 490                 int i = 0;
 491
 492                 for(i = 0; i < mali_platform_device->num_resources; i++)
 493                         dev_info(&pdev->dev, "mali_platform_device->resource[%d].start = 0x%08x\n",
 494                                 i, mali_platform_device->resource[i].start);
 495         }
 496
 497 #ifdef CONFIG_MALI_DT
 498         /* If we use DT to initialize our DDK, we have to prepare somethings. */
 499         err = mali_platform_device_init(mali_platform_device);
 500         if (0 != err) {
 501                 MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device."));
 502                 return -EFAULT;
 503         }
 504 #endif
 505
 506         if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) {
 507                 /* Initialize the Mali GPU HW specified by pdev */
 508                 if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) {
 509                         /* Register a misc device (so we are accessible from user space) */
 510                         err = mali_miscdevice_register(pdev);
 511                         if (0 == err) {
 512                                 /* Setup sysfs entries */
 513                                 err = mali_sysfs_register(mali_dev_name);
 514
 515                                 if (0 == err) {
 516                                         MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name));
 517
 518                                         return 0;
 519                                 } else {
 520                                         MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries"));
 521                                 }
 522                                 mali_miscdevice_unregister();
 523                         } else {
 524                                 MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device."));
 525                         }
 526                         mali_terminate_subsystems();
 527                 } else {
 528                         MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver."));
 529                 }
 530                 _mali_osk_wq_term();
 531         }
 532
 533         mali_platform_device = NULL;
 534         return -EFAULT;
 535 }
 536
 537 static int mali_remove(struct platform_device *pdev)
 538 {
 539         MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name));
 540         mali_sysfs_unregister();
 541         mali_miscdevice_unregister();
 542         mali_terminate_subsystems();
 543         _mali_osk_wq_term();
 544 #ifdef CONFIG_MALI_DT
 545         mali_platform_device_deinit(mali_platform_device);
 546 #endif
 547         mali_platform_device = NULL;
 548         return 0;
 549 }
 550
 551 static int mali_miscdevice_register(struct platform_device *pdev)
 552 {
 553         int err;
 554
 555         mali_miscdevice.minor = MISC_DYNAMIC_MINOR;
 556         mali_miscdevice.name = mali_dev_name;
 557         mali_miscdevice.fops = &mali_fops;
 558         mali_miscdevice.parent = get_device(&pdev->dev);
 559
 560         err = misc_register(&mali_miscdevice);
 561         if (0 != err) {
 562                 MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err));
 563         }
 564
 565         return err;
 566 }
 567
 568 static void mali_miscdevice_unregister(void)
 569 {
 570         misc_deregister(&mali_miscdevice);
 571 }
 572
 573 static int mali_driver_suspend_scheduler(struct device *dev)
 574 {
 575         mali_pm_os_suspend(MALI_TRUE);
 576         /* Tracing the frequency and voltage after mali is suspended */
 577         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 578                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 579                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 580                                       0,
 581                                       0,
 582                                       0, 0, 0);
 583         return 0;
 584 }
 585
 586 static int mali_driver_resume_scheduler(struct device *dev)
 587 {
 588         /* Tracing the frequency and voltage after mali is resumed */
 589 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 590         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 591         if (is_first_resume == 1) {
 592                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 593                 is_first_resume = 0;
 594         }
 595         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 596                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 597                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 598                                       mali_gpu_clk[1].clock,
 599                                       mali_gpu_clk[1].vol / 1000,
 600                                       0, 0, 0);
 601 #endif
 602         mali_pm_os_resume();
 603         return 0;
 604 }
 605
 606 #ifdef CONFIG_PM_RUNTIME
 607 static int mali_driver_runtime_suspend(struct device *dev)
 608 {
 609         if (MALI_TRUE == mali_pm_runtime_suspend()) {
 610                 /* Tracing the frequency and voltage after mali is suspended */
 611                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 612                                               MALI_PROFILING_EVENT_CHANNEL_GPU |
 613                                               MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 614                                               0,
 615                                               0,
 616                                               0, 0, 0);
 617
 618                 return 0;
 619         } else {
 620                 return -EBUSY;
 621         }
 622 }
 623
 624 static int mali_driver_runtime_resume(struct device *dev)
 625 {
 626         /* Tracing the frequency and voltage after mali is resumed */
 627 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 628         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 629         if (is_first_resume == 1) {
 630                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 631                 is_first_resume = 0;
 632         }
 633         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 634                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 635                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 636                                       mali_gpu_clk[1].clock,
 637                                       mali_gpu_clk[1].vol / 1000,
 638                                       0, 0, 0);
 639 #endif
 640
 641         mali_pm_runtime_resume();
 642         return 0;
 643 }
 644
 645 static int mali_driver_runtime_idle(struct device *dev)
 646 {
 647         /* Nothing to do */
 648         return 0;
 649 }
 650 #endif
 651
 652 static int mali_open(struct inode *inode, struct file *filp)
 653 {
 654         struct mali_session_data *session_data;
 655         _mali_osk_errcode_t err;
 656
 657         /* input validation */
 658         if (mali_miscdevice.minor != iminor(inode)) {
 659                 MALI_PRINT_ERROR(("mali_open() Minor does not match\n"));
 660                 return -ENODEV;
 661         }
 662
 663         /* allocated struct to track this session */
 664         err = _mali_ukk_open((void **)&session_data);
 665         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 666
 667         /* initialize file pointer */
 668         filp->f_pos = 0;
 669
 670         /* link in our session data */
 671         filp->private_data = (void *)session_data;
 672
 673         return 0;
 674 }
 675
 676 static int mali_release(struct inode *inode, struct file *filp)
 677 {
 678         _mali_osk_errcode_t err;
 679
 680         /* input validation */
 681         if (mali_miscdevice.minor != iminor(inode)) {
 682                 MALI_PRINT_ERROR(("mali_release() Minor does not match\n"));
 683                 return -ENODEV;
 684         }
 685
 686         err = _mali_ukk_close((void **)&filp->private_data);
 687         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 688
 689         return 0;
 690 }
 691
 692 int map_errcode(_mali_osk_errcode_t err)
 693 {
 694         switch (err) {
 695         case _MALI_OSK_ERR_OK :
 696                 return 0;
 697         case _MALI_OSK_ERR_FAULT:
 698                 return -EFAULT;
 699         case _MALI_OSK_ERR_INVALID_FUNC:
 700                 return -ENOTTY;
 701         case _MALI_OSK_ERR_INVALID_ARGS:
 702                 return -EINVAL;
 703         case _MALI_OSK_ERR_NOMEM:
 704                 return -ENOMEM;
 705         case _MALI_OSK_ERR_TIMEOUT:
 706                 return -ETIMEDOUT;
 707         case _MALI_OSK_ERR_RESTARTSYSCALL:
 708                 return -ERESTARTSYS;
 709         case _MALI_OSK_ERR_ITEM_NOT_FOUND:
 710                 return -ENOENT;
 711         default:
 712                 return -EFAULT;
 713         }
 714 }
 715
 716 #ifdef HAVE_UNLOCKED_IOCTL
 717 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 718 #else
 719 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
 720 #endif
 721 {
 722         int err;
 723         struct mali_session_data *session_data;
 724
 725 #ifndef HAVE_UNLOCKED_IOCTL
 726         /* inode not used */
 727         (void)inode;
 728 #endif
 729
 730         MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg));
 731
 732         session_data = (struct mali_session_data *)filp->private_data;
 733         if (NULL == session_data) {
 734                 MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n"));
 735                 return -ENOTTY;
 736         }
 737
 738         if (NULL == (void *)arg) {
 739                 MALI_DEBUG_PRINT(7, ("arg was NULL\n"));
 740                 return -ENOTTY;
 741         }
 742
 743         switch (cmd) {
 744         case MALI_IOC_WAIT_FOR_NOTIFICATION:
 745                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64)));
 746                 err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg);
 747                 break;
 748
 749         case MALI_IOC_GET_API_VERSION_V2:
 750                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64)));
 751                 err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg);
 752                 break;
 753
 754         case MALI_IOC_GET_API_VERSION:
 755                 err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg);
 756                 break;
 757
 758         case MALI_IOC_POST_NOTIFICATION:
 759                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64)));
 760                 err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg);
 761                 break;
 762
 763     /* rk_ext : 从对 r5p0-01rel0 集成开始, 不再使用. */
 764 #if 0
 765         case MALI_IOC_GET_MALI_VERSION_IN_RK30:
 766                 err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg);
 767                 break;
 768 #else
 769     case MALI_IOC_GET_RK_KO_VERSION:
 770                 err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg);
 771                 break;
 772 #endif
 773
 774         case MALI_IOC_GET_USER_SETTINGS:
 775                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64)));
 776                 err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg);
 777                 break;
 778
 779         case MALI_IOC_REQUEST_HIGH_PRIORITY:
 780                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64)));
 781                 err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg);
 782                 break;
 783
 784 #if defined(CONFIG_MALI400_PROFILING)
 785         case MALI_IOC_PROFILING_ADD_EVENT:
 786                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64)));
 787                 err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg);
 788                 break;
 789
 790         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS:
 791                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64)));
 792                 err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg);
 793                 break;
 794
 795
 796         case MALI_IOC_PROFILING_MEMORY_USAGE_GET:
 797                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64)));
 798                 err = profiling_memory_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg);
 799                 break;
 800
 801 #else
 802
 803         case MALI_IOC_PROFILING_ADD_EVENT:          /* FALL-THROUGH */
 804         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */
 805         case MALI_IOC_PROFILING_MEMORY_USAGE_GET:   /* FALL-THROUGH */
 806                 MALI_DEBUG_PRINT(2, ("Profiling not supported\n"));
 807                 err = -ENOTTY;
 808                 break;
 809
 810 #endif
 811
 812         case MALI_IOC_MEM_WRITE_SAFE:
 813                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64)));
 814                 err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg);
 815                 break;
 816
 817         case MALI_IOC_MEM_MAP_EXT:
 818                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_map_external_mem_s), sizeof(u64)));
 819                 err = mem_map_ext_wrapper(session_data, (_mali_uk_map_external_mem_s __user *)arg);
 820                 break;
 821
 822         case MALI_IOC_MEM_UNMAP_EXT:
 823                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unmap_external_mem_s), sizeof(u64)));
 824                 err = mem_unmap_ext_wrapper(session_data, (_mali_uk_unmap_external_mem_s __user *)arg);
 825                 break;
 826
 827         case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE:
 828                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64)));
 829                 err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg);
 830                 break;
 831
 832         case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE:
 833                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64)));
 834                 err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg);
 835                 break;
 836
 837 #if defined(CONFIG_MALI400_UMP)
 838
 839         case MALI_IOC_MEM_ATTACH_UMP:
 840                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_ump_mem_s), sizeof(u64)));
 841                 err = mem_attach_ump_wrapper(session_data, (_mali_uk_attach_ump_mem_s __user *)arg);
 842                 break;
 843
 844         case MALI_IOC_MEM_RELEASE_UMP:
 845                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_ump_mem_s), sizeof(u64)));
 846                 err = mem_release_ump_wrapper(session_data, (_mali_uk_release_ump_mem_s __user *)arg);
 847                 break;
 848
 849 #else
 850
 851         case MALI_IOC_MEM_ATTACH_UMP:
 852         case MALI_IOC_MEM_RELEASE_UMP: /* FALL-THROUGH */
 853                 MALI_DEBUG_PRINT(2, ("UMP not supported\n"));
 854                 err = -ENOTTY;
 855                 break;
 856 #endif
 857
 858 #ifdef CONFIG_DMA_SHARED_BUFFER
 859         case MALI_IOC_MEM_ATTACH_DMA_BUF:
 860                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_dma_buf_s), sizeof(u64)));
 861                 err = mali_attach_dma_buf(session_data, (_mali_uk_attach_dma_buf_s __user *)arg);
 862                 break;
 863
 864         case MALI_IOC_MEM_RELEASE_DMA_BUF:
 865                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_dma_buf_s), sizeof(u64)));
 866                 err = mali_release_dma_buf(session_data, (_mali_uk_release_dma_buf_s __user *)arg);
 867                 break;
 868
 869         case MALI_IOC_MEM_DMA_BUF_GET_SIZE:
 870                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64)));
 871                 err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg);
 872                 break;
 873 #else
 874
 875         case MALI_IOC_MEM_ATTACH_DMA_BUF:   /* FALL-THROUGH */
 876         case MALI_IOC_MEM_RELEASE_DMA_BUF:  /* FALL-THROUGH */
 877         case MALI_IOC_MEM_DMA_BUF_GET_SIZE: /* FALL-THROUGH */
 878                 MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n"));
 879                 err = -ENOTTY;
 880                 break;
 881 #endif
 882
 883         case MALI_IOC_PP_START_JOB:
 884                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64)));
 885                 err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg);
 886                 break;
 887
 888         case MALI_IOC_PP_AND_GP_START_JOB:
 889                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64)));
 890                 err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg);
 891                 break;
 892
 893         case MALI_IOC_PP_NUMBER_OF_CORES_GET:
 894                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64)));
 895                 err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg);
 896                 break;
 897
 898         case MALI_IOC_PP_CORE_VERSION_GET:
 899                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64)));
 900                 err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg);
 901                 break;
 902
 903         case MALI_IOC_PP_DISABLE_WB:
 904                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64)));
 905                 err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg);
 906                 break;
 907
 908         case MALI_IOC_GP2_START_JOB:
 909                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64)));
 910                 err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg);
 911                 break;
 912
 913         case MALI_IOC_GP2_NUMBER_OF_CORES_GET:
 914                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64)));
 915                 err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg);
 916                 break;
 917
 918         case MALI_IOC_GP2_CORE_VERSION_GET:
 919                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64)));
 920                 err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg);
 921                 break;
 922
 923         case MALI_IOC_GP2_SUSPEND_RESPONSE:
 924                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64)));
 925                 err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg);
 926                 break;
 927
 928         case MALI_IOC_VSYNC_EVENT_REPORT:
 929                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64)));
 930                 err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg);
 931                 break;
 932
 933         case MALI_IOC_TIMELINE_GET_LATEST_POINT:
 934                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64)));
 935                 err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg);
 936                 break;
 937         case MALI_IOC_TIMELINE_WAIT:
 938                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64)));
 939                 err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg);
 940                 break;
 941         case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE:
 942                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64)));
 943                 err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg);
 944                 break;
 945         case MALI_IOC_SOFT_JOB_START:
 946                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64)));
 947                 err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg);
 948                 break;
 949         case MALI_IOC_SOFT_JOB_SIGNAL:
 950                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64)));
 951                 err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg);
 952                 break;
 953
 954         default:
 955                 MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg));
 956                 err = -ENOTTY;
 957         };
 958
 959         return err;
 960 }
 961
 962
 963 module_init(mali_module_init);
 964 module_exit(mali_module_exit);
 965
 966 MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE);
 967 MODULE_AUTHOR("ARM Ltd.");
 968 MODULE_VERSION(SVN_REV_STRING);