arch/arm/mach-rockchip/vcodec_service.c

   1
   2 /* arch/arm/mach-rk29/vpu.c
   3  *
   4  * Copyright (C) 2010 ROCKCHIP, Inc.
   5  * author: chenhengming chm@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #include <linux/clk.h>
  19 #include <linux/delay.h>
  20 #include <linux/init.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io.h>
  23 #include <linux/kernel.h>
  24 #include <linux/module.h>
  25 #include <linux/fs.h>
  26 #include <linux/ioport.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/mm.h>
  29 #include <linux/poll.h>
  30 #include <linux/platform_device.h>
  31 #include <linux/sched.h>
  32 #include <linux/slab.h>
  33 #include <linux/wakelock.h>
  34 #include <linux/cdev.h>
  35 #include <linux/of.h>
  36 #include <linux/of_platform.h>
  37 #include <linux/rockchip/cpu.h>
  38 #include <linux/rockchip/cru.h>
  39
  40 #include <asm/cacheflush.h>
  41 #include <linux/uaccess.h>
  42 #include <linux/rockchip/grf.h>
  43
  44 #if defined(CONFIG_ION_ROCKCHIP)
  45 #include <linux/rockchip_ion.h>
  46 #endif
  47
  48 #if defined(CONFIG_ROCKCHIP_IOMMU) & defined(CONFIG_ION_ROCKCHIP)
  49 #define CONFIG_VCODEC_MMU
  50 #endif
  51
  52 #ifdef CONFIG_VCODEC_MMU
  53 #include <linux/rockchip-iovmm.h>
  54 #include <linux/dma-buf.h>
  55 #endif
  56
  57 #ifdef CONFIG_DEBUG_FS
  58 #include <linux/debugfs.h>
  59 #endif
  60
  61 #if defined(CONFIG_ARCH_RK319X)
  62 #include <mach/grf.h>
  63 #endif
  64
  65 #include "vcodec_service.h"
  66
  67 #define HEVC_TEST_ENABLE        0
  68 #define HEVC_SIM_ENABLE         0
  69 #define VCODEC_CLOCK_ENABLE     1
  70
  71 typedef enum {
  72         VPU_DEC_ID_9190         = 0x6731,
  73         VPU_ID_8270             = 0x8270,
  74         VPU_ID_4831             = 0x4831,
  75         HEVC_ID                 = 0x6867,
  76 } VPU_HW_ID;
  77
  78 typedef enum {
  79         VPU_DEC_TYPE_9190       = 0,
  80         VPU_ENC_TYPE_8270       = 0x100,
  81         VPU_ENC_TYPE_4831       ,
  82 } VPU_HW_TYPE_E;
  83
  84 typedef enum VPU_FREQ {
  85         VPU_FREQ_200M,
  86         VPU_FREQ_266M,
  87         VPU_FREQ_300M,
  88         VPU_FREQ_400M,
  89         VPU_FREQ_500M,
  90         VPU_FREQ_600M,
  91         VPU_FREQ_DEFAULT,
  92         VPU_FREQ_BUT,
  93 } VPU_FREQ;
  94
  95 typedef struct {
  96         VPU_HW_ID               hw_id;
  97         unsigned long           hw_addr;
  98         unsigned long           enc_offset;
  99         unsigned long           enc_reg_num;
 100         unsigned long           enc_io_size;
 101         unsigned long           dec_offset;
 102         unsigned long           dec_reg_num;
 103         unsigned long           dec_io_size;
 104 } VPU_HW_INFO_E;
 105
 106 struct extra_info_elem {
 107         u32 index;
 108         u32 offset;
 109 };
 110
 111 #define EXTRA_INFO_MAGIC        0x4C4A46
 112
 113 struct extra_info_for_iommu {
 114         u32 magic;
 115         u32 cnt;
 116         struct extra_info_elem elem[20];
 117 };
 118
 119 #define VPU_SERVICE_SHOW_TIME                   0
 120
 121 #if VPU_SERVICE_SHOW_TIME
 122 static struct timeval enc_start, enc_end;
 123 static struct timeval dec_start, dec_end;
 124 static struct timeval pp_start,  pp_end;
 125 #endif
 126
 127 #define MHZ                                     (1000*1000)
 128
 129 #define REG_NUM_9190_DEC                        (60)
 130 #define REG_NUM_9190_PP                         (41)
 131 #define REG_NUM_9190_DEC_PP                     (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 132
 133 #define REG_NUM_DEC_PP                          (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 134
 135 #define REG_NUM_ENC_8270                        (96)
 136 #define REG_SIZE_ENC_8270                       (0x200)
 137 #define REG_NUM_ENC_4831                        (164)
 138 #define REG_SIZE_ENC_4831                       (0x400)
 139
 140 #define REG_NUM_HEVC_DEC                        (68)
 141
 142 #define SIZE_REG(reg)                           ((reg)*4)
 143
 144 static VPU_HW_INFO_E vpu_hw_set[] = {
 145         [0] = {
 146                 .hw_id          = VPU_ID_8270,
 147                 .hw_addr        = 0,
 148                 .enc_offset     = 0x0,
 149                 .enc_reg_num    = REG_NUM_ENC_8270,
 150                 .enc_io_size    = REG_NUM_ENC_8270 * 4,
 151                 .dec_offset     = REG_SIZE_ENC_8270,
 152                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 153                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 154         },
 155         [1] = {
 156                 .hw_id          = VPU_ID_4831,
 157                 .hw_addr        = 0,
 158                 .enc_offset     = 0x0,
 159                 .enc_reg_num    = REG_NUM_ENC_4831,
 160                 .enc_io_size    = REG_NUM_ENC_4831 * 4,
 161                 .dec_offset     = REG_SIZE_ENC_4831,
 162                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 163                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 164         },
 165         [2] = {
 166                 .hw_id          = HEVC_ID,
 167                 .hw_addr        = 0,
 168                 .dec_offset     = 0x0,
 169                 .dec_reg_num    = REG_NUM_HEVC_DEC,
 170                 .dec_io_size    = REG_NUM_HEVC_DEC * 4,
 171         },
 172         [3] = {
 173                 .hw_id          = VPU_DEC_ID_9190,
 174                 .hw_addr        = 0,
 175                 .enc_offset     = 0x0,
 176                 .enc_reg_num    = 0,
 177                 .enc_io_size    = 0,
 178                 .dec_offset     = REG_SIZE_ENC_4831,
 179                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 180                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 181         },
 182
 183 };
 184
 185
 186 #define DEC_INTERRUPT_REGISTER                  1
 187 #define PP_INTERRUPT_REGISTER                   60
 188 #define ENC_INTERRUPT_REGISTER                  1
 189
 190 #define DEC_INTERRUPT_BIT                       0x100
 191 #define DEC_BUFFER_EMPTY_BIT                    0x4000
 192 #define PP_INTERRUPT_BIT                        0x100
 193 #define ENC_INTERRUPT_BIT                       0x1
 194
 195 #define HEVC_DEC_INT_RAW_BIT                    0x200
 196 #define HEVC_DEC_STR_ERROR_BIT                  0x4000
 197 #define HEVC_DEC_BUS_ERROR_BIT                  0x2000
 198 #define HEVC_DEC_BUFFER_EMPTY_BIT               0x10000
 199
 200 #define VPU_REG_EN_ENC                          14
 201 #define VPU_REG_ENC_GATE                        2
 202 #define VPU_REG_ENC_GATE_BIT                    (1<<4)
 203
 204 #define VPU_REG_EN_DEC                          1
 205 #define VPU_REG_DEC_GATE                        2
 206 #define VPU_REG_DEC_GATE_BIT                    (1<<10)
 207 #define VPU_REG_EN_PP                           0
 208 #define VPU_REG_PP_GATE                         1
 209 #define VPU_REG_PP_GATE_BIT                     (1<<8)
 210 #define VPU_REG_EN_DEC_PP                       1
 211 #define VPU_REG_DEC_PP_GATE                     61
 212 #define VPU_REG_DEC_PP_GATE_BIT                 (1<<8)
 213
 214 #if defined(CONFIG_VCODEC_MMU)
 215 static u8 addr_tbl_vpu_h264dec[] = {
 216         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
 217         25, 26, 27, 28, 29, 40, 41
 218 };
 219
 220 static u8 addr_tbl_vpu_vp8dec[] = {
 221         10, 12, 13, 14, 18, 19, 27, 40
 222 };
 223
 224 static u8 addr_tbl_vpu_vp6dec[] = {
 225         12, 13, 14, 18, 27, 40
 226 };
 227
 228 static u8 addr_tbl_vpu_vc1dec[] = {
 229         12, 13, 14, 15, 16, 17, 27, 41
 230 };
 231
 232 static u8 addr_tbl_vpu_jpegdec[] = {
 233         12, 40, 66, 67
 234 };
 235
 236 static u8 addr_tbl_vpu_defaultdec[] = {
 237         12, 13, 14, 15, 16, 17, 40, 41
 238 };
 239
 240 static u8 addr_tbl_vpu_enc[] = {
 241         5, 6, 7, 8, 9, 10, 11, 12, 13, 51
 242 };
 243
 244 static u8 addr_tbl_hevc_dec[] = {
 245         4, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
 246         21, 22, 23, 24, 42, 43
 247 };
 248 #endif
 249
 250 enum VPU_DEC_FMT {
 251         VPU_DEC_FMT_H264,
 252         VPU_DEC_FMT_MPEG4,
 253         VPU_DEC_FMT_H263,
 254         VPU_DEC_FMT_JPEG,
 255         VPU_DEC_FMT_VC1,
 256         VPU_DEC_FMT_MPEG2,
 257         VPU_DEC_FMT_MPEG1,
 258         VPU_DEC_FMT_VP6,
 259         VPU_DEC_FMT_RV,
 260         VPU_DEC_FMT_VP7,
 261         VPU_DEC_FMT_VP8,
 262         VPU_DEC_FMT_AVS,
 263         VPU_DEC_FMT_SVC,
 264         VPU_DEC_FMT_VC2,
 265         VPU_DEC_FMT_MVC,
 266         VPU_DEC_FMT_THEORA,
 267         VPU_DEC_FMT_RES
 268 };
 269
 270 /**
 271  * struct for process session which connect to vpu
 272  *
 273  * @author ChenHengming (2011-5-3)
 274  */
 275 typedef struct vpu_session {
 276         VPU_CLIENT_TYPE         type;
 277         /* a linked list of data so we can access them for debugging */
 278         struct list_head        list_session;
 279         /* a linked list of register data waiting for process */
 280         struct list_head        waiting;
 281         /* a linked list of register data in processing */
 282         struct list_head        running;
 283         /* a linked list of register data processed */
 284         struct list_head        done;
 285         wait_queue_head_t       wait;
 286         pid_t                   pid;
 287         atomic_t                task_running;
 288 } vpu_session;
 289
 290 /**
 291  * struct for process register set
 292  *
 293  * @author ChenHengming (2011-5-4)
 294  */
 295 typedef struct vpu_reg {
 296         VPU_CLIENT_TYPE         type;
 297         VPU_FREQ                freq;
 298         vpu_session             *session;
 299         struct list_head        session_link;           /* link to vpu service session */
 300         struct list_head        status_link;            /* link to register set list */
 301         unsigned long           size;
 302 #if defined(CONFIG_VCODEC_MMU)
 303         struct list_head        mem_region_list;
 304 #endif
 305         unsigned long           *reg;
 306 } vpu_reg;
 307
 308 typedef struct vpu_device {
 309         atomic_t                irq_count_codec;
 310         atomic_t                irq_count_pp;
 311         unsigned long           iobaseaddr;
 312         unsigned int            iosize;
 313         volatile u32            *hwregs;
 314 } vpu_device;
 315
 316 enum vcodec_device_id {
 317         VCODEC_DEVICE_ID_VPU,
 318         VCODEC_DEVICE_ID_HEVC
 319 };
 320
 321 struct vcodec_mem_region {
 322         struct list_head srv_lnk;
 323         struct list_head reg_lnk;
 324         struct list_head session_lnk;
 325         unsigned long iova;     /* virtual address for iommu */
 326         unsigned long len;
 327         struct ion_handle *hdl;
 328 };
 329
 330 typedef struct vpu_service_info {
 331         struct wake_lock        wake_lock;
 332         struct delayed_work     power_off_work;
 333         struct mutex            lock;
 334         struct list_head        waiting;                /* link to link_reg in struct vpu_reg */
 335         struct list_head        running;                /* link to link_reg in struct vpu_reg */
 336         struct list_head        done;                   /* link to link_reg in struct vpu_reg */
 337         struct list_head        session;                /* link to list_session in struct vpu_session */
 338         atomic_t                total_running;
 339         bool                    enabled;
 340         vpu_reg                 *reg_codec;
 341         vpu_reg                 *reg_pproc;
 342         vpu_reg                 *reg_resev;
 343         VPUHwDecConfig_t        dec_config;
 344         VPUHwEncConfig_t        enc_config;
 345         VPU_HW_INFO_E           *hw_info;
 346         unsigned long           reg_size;
 347         bool                    auto_freq;
 348         bool                    bug_dec_addr;
 349         atomic_t                freq_status;
 350
 351         struct clk              *aclk_vcodec;
 352         struct clk              *hclk_vcodec;
 353         struct clk              *clk_core;
 354         struct clk              *clk_cabac;
 355         struct clk              *pd_video;
 356
 357         int                     irq_dec;
 358         int                     irq_enc;
 359
 360         vpu_device              enc_dev;
 361         vpu_device              dec_dev;
 362
 363         struct device           *dev;
 364
 365         struct cdev             cdev;
 366         dev_t                   dev_t;
 367         struct class            *cls;
 368         struct device           *child_dev;
 369
 370         struct dentry           *debugfs_dir;
 371         struct dentry           *debugfs_file_regs;
 372
 373         u32 irq_status;
 374 #if defined(CONFIG_VCODEC_MMU)
 375         struct ion_client       *ion_client;
 376         struct list_head        mem_region_list;
 377         struct device           *mmu_dev;
 378 #endif
 379
 380         enum vcodec_device_id   dev_id;
 381
 382         u32                     reserved_mode;
 383
 384         struct delayed_work     simulate_work;
 385 } vpu_service_info;
 386
 387 typedef struct vpu_request
 388 {
 389         unsigned long *req;
 390         unsigned long size;
 391 } vpu_request;
 392
 393 /// global variable
 394 //static struct clk *pd_video;
 395 static struct dentry *parent; // debugfs root directory for all device (vpu, hevc).
 396 /* mutex for selecting operation registers of vpu or hevc */
 397 static struct mutex g_mode_mutex;
 398
 399 #ifdef CONFIG_DEBUG_FS
 400 static int vcodec_debugfs_init(void);
 401 static void vcodec_debugfs_exit(void);
 402 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent);
 403 static int debug_vcodec_open(struct inode *inode, struct file *file);
 404
 405 static const struct file_operations debug_vcodec_fops = {
 406         .open = debug_vcodec_open,
 407         .read = seq_read,
 408         .llseek = seq_lseek,
 409         .release = single_release,
 410 };
 411 #endif
 412
 413 #define VDPU_SOFT_RESET_REG     101
 414 #define VDPU_CLEAR_CACHE_REG    516
 415 #define VEPU_CLEAR_CACHE_REG    772
 416 #define HEVC_CLEAR_CACHE_REG    260
 417
 418 #define VPU_REG_ENABLE(base, reg)       do { \
 419                                                 base[reg] = 1; \
 420                                         } while (0)
 421
 422 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 423 #define VDPU_CLEAR_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAR_CACHE_REG)
 424 #define VEPU_CLEAR_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAR_CACHE_REG)
 425 #define HEVC_CLEAR_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAR_CACHE_REG)
 426
 427 #define VPU_POWER_OFF_DELAY             4*HZ /* 4s */
 428 #define VPU_TIMEOUT_DELAY               2*HZ /* 2s */
 429 #define VPU_SIMULATE_DELAY              msecs_to_jiffies(15)
 430
 431 #define BIT_VCODEC_SEL_RK3036           (1<<3)
 432 #define BIT_VCODEC_SEL_RK312X           (1<<15)
 433 static void vcodec_enter_mode_nolock(enum vcodec_device_id id, u32 *reserved_mode)
 434 {
 435         if (cpu_is_rk3036() || cpu_is_rk312x()) {
 436                 int bits = cpu_is_rk3036() ? BIT_VCODEC_SEL_RK3036 : BIT_VCODEC_SEL_RK312X;
 437                 void __iomem *addr = cpu_is_rk3036() ? (RK_GRF_VIRT + RK3036_GRF_SOC_CON1) : (RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 438                 if (reserved_mode)
 439                         *reserved_mode = readl_relaxed(addr);
 440                 if (id == VCODEC_DEVICE_ID_HEVC)
 441                         writel_relaxed(readl_relaxed(addr) | (bits) | (bits << 16), addr);
 442                 else
 443                         writel_relaxed((readl_relaxed(addr) & (~bits)) | (bits << 16), addr);
 444         }
 445 }
 446
 447 static void vcodec_exit_mode_nolock(enum vcodec_device_id id, u32 reserved_mode)
 448 {
 449         if (cpu_is_rk3036() || cpu_is_rk312x()) {
 450                 int bits = cpu_is_rk3036() ? BIT_VCODEC_SEL_RK3036 : BIT_VCODEC_SEL_RK312X;
 451                 void __iomem *addr = cpu_is_rk3036() ? (RK_GRF_VIRT + RK3036_GRF_SOC_CON1) : (RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 452                 writel_relaxed(reserved_mode | (bits << 16), addr);
 453         }
 454 }
 455
 456 static void vcodec_enter_mode(enum vcodec_device_id id)
 457 {
 458         if (cpu_is_rk3036() || cpu_is_rk312x())
 459                 mutex_lock(&g_mode_mutex);
 460         vcodec_enter_mode_nolock(id, NULL);
 461 }
 462
 463 static void vcodec_exit_mode(void)
 464 {
 465         if (cpu_is_rk3036() || cpu_is_rk312x())
 466                 mutex_unlock(&g_mode_mutex);
 467 }
 468
 469 static int vpu_get_clk(struct vpu_service_info *pservice)
 470 {
 471 #if VCODEC_CLOCK_ENABLE
 472         do {
 473                 pservice->aclk_vcodec   = devm_clk_get(pservice->dev, "aclk_vcodec");
 474                 if (IS_ERR(pservice->aclk_vcodec)) {
 475                         dev_err(pservice->dev, "failed on clk_get aclk_vcodec\n");
 476                         break;
 477                 }
 478
 479                 pservice->hclk_vcodec   = devm_clk_get(pservice->dev, "hclk_vcodec");
 480                 if (IS_ERR(pservice->hclk_vcodec)) {
 481                         dev_err(pservice->dev, "failed on clk_get hclk_vcodec\n");
 482                         break;
 483                 }
 484
 485                 if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC) {
 486                         pservice->clk_core = devm_clk_get(pservice->dev, "clk_core");
 487                         if (IS_ERR(pservice->clk_core)) {
 488                                 dev_err(pservice->dev, "failed on clk_get clk_core\n");
 489                                 break;
 490                         }
 491
 492                         if (!cpu_is_rk3036() && !cpu_is_rk312x()) {
 493                                 pservice->clk_cabac = devm_clk_get(pservice->dev, "clk_cabac");
 494                                 if (IS_ERR(pservice->clk_cabac)) {
 495                                         dev_err(pservice->dev, "failed on clk_get clk_cabac\n");
 496                                         break;
 497                                 }
 498                         } else {
 499                                 pservice->clk_cabac = NULL;
 500                         }
 501
 502                         if (!cpu_is_rk3036() && !cpu_is_rk312x()) {
 503                                 pservice->pd_video = devm_clk_get(pservice->dev, "pd_hevc");
 504                                 if (IS_ERR(pservice->pd_video)) {
 505                                         dev_err(pservice->dev, "failed on clk_get pd_hevc\n");
 506                                         break;
 507                                 }
 508                         } else {
 509                                 pservice->pd_video = NULL;
 510                         }
 511                 } else {
 512                         if (!cpu_is_rk3036() && !cpu_is_rk312x()) {
 513                                 pservice->pd_video = devm_clk_get(pservice->dev, "pd_video");
 514                                 if (IS_ERR(pservice->pd_video)) {
 515                                         dev_err(pservice->dev, "failed on clk_get pd_video\n");
 516                                         break;
 517                                 }
 518                         } else {
 519                                 pservice->pd_video = NULL;
 520                         }
 521                 }
 522
 523                 if (cpu_is_rk312x()) {
 524                         pservice->pd_video = devm_clk_get(pservice->dev, "pd_video");
 525                         if (IS_ERR(pservice->pd_video)) {
 526                                 dev_err(pservice->dev, "failed on clk_get pd_video\n");
 527                                 break;
 528                         }
 529                 }
 530
 531                 return 0;
 532         } while (0);
 533
 534         return -1;
 535 #else
 536         return 0;
 537 #endif
 538 }
 539
 540 static void vpu_put_clk(struct vpu_service_info *pservice)
 541 {
 542 #if VCODEC_CLOCK_ENABLE
 543         if (pservice->pd_video) {
 544                 devm_clk_put(pservice->dev, pservice->pd_video);
 545         }
 546
 547         if (pservice->aclk_vcodec) {
 548                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 549         }
 550
 551         if (pservice->hclk_vcodec) {
 552                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 553         }
 554
 555         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC) {
 556                 if (pservice->clk_core) {
 557                         devm_clk_put(pservice->dev, pservice->clk_core);
 558                 }
 559
 560                 if (pservice->clk_cabac) {
 561                         devm_clk_put(pservice->dev, pservice->clk_cabac);
 562                 }
 563         }
 564 #endif
 565 }
 566
 567 static void vpu_reset(struct vpu_service_info *pservice)
 568 {
 569 #if defined(CONFIG_ARCH_RK29)
 570         clk_disable(aclk_ddr_vepu);
 571         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 572         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 573         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 574         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 575         mdelay(10);
 576         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 577         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 578         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 579         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 580         clk_enable(aclk_ddr_vepu);
 581 #elif defined(CONFIG_ARCH_RK30)
 582         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 583         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 584         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 585         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 586         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 587         mdelay(1);
 588         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 589         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 590         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 591         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 592         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 593 #endif
 594
 595         pservice->reg_codec = NULL;
 596         pservice->reg_pproc = NULL;
 597         pservice->reg_resev = NULL;
 598 }
 599
 600 static void reg_deinit(struct vpu_service_info *pservice, vpu_reg *reg);
 601 static void vpu_service_session_clear(struct vpu_service_info *pservice, vpu_session *session)
 602 {
 603         vpu_reg *reg, *n;
 604         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 605                 reg_deinit(pservice, reg);
 606         }
 607         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 608                 reg_deinit(pservice, reg);
 609         }
 610         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 611                 reg_deinit(pservice, reg);
 612         }
 613 }
 614
 615 static void vpu_service_dump(struct vpu_service_info *pservice)
 616 {
 617         int running;
 618         vpu_reg *reg, *reg_tmp;
 619         vpu_session *session, *session_tmp;
 620
 621         running = atomic_read(&pservice->total_running);
 622         printk("total_running %d\n", running);
 623
 624         printk("reg_codec 0x%.8x\n", (unsigned int)pservice->reg_codec);
 625         printk("reg_pproc 0x%.8x\n", (unsigned int)pservice->reg_pproc);
 626         printk("reg_resev 0x%.8x\n", (unsigned int)pservice->reg_resev);
 627
 628         list_for_each_entry_safe(session, session_tmp, &pservice->session, list_session) {
 629                 printk("session pid %d type %d:\n", session->pid, session->type);
 630                 running = atomic_read(&session->task_running);
 631                 printk("task_running %d\n", running);
 632                 list_for_each_entry_safe(reg, reg_tmp, &session->waiting, session_link) {
 633                         printk("waiting register set 0x%.8x\n", (unsigned int)reg);
 634                 }
 635                 list_for_each_entry_safe(reg, reg_tmp, &session->running, session_link) {
 636                         printk("running register set 0x%.8x\n", (unsigned int)reg);
 637                 }
 638                 list_for_each_entry_safe(reg, reg_tmp, &session->done, session_link) {
 639                         printk("done    register set 0x%.8x\n", (unsigned int)reg);
 640                 }
 641         }
 642 }
 643
 644 static void vpu_service_power_off(struct vpu_service_info *pservice)
 645 {
 646         int total_running;
 647         if (!pservice->enabled)
 648                 return;
 649
 650         pservice->enabled = false;
 651         total_running = atomic_read(&pservice->total_running);
 652         if (total_running) {
 653                 pr_alert("alert: power off when %d task running!!\n", total_running);
 654                 mdelay(50);
 655                 pr_alert("alert: delay 50 ms for running task\n");
 656                 vpu_service_dump(pservice);
 657         }
 658
 659         pr_info("%s: power off...", dev_name(pservice->dev));
 660         udelay(10);
 661
 662 #if defined(CONFIG_VCODEC_MMU)
 663         if (pservice->mmu_dev) {
 664                 vcodec_enter_mode(pservice->dev_id);
 665                 rockchip_iovmm_deactivate(pservice->dev);
 666                 vcodec_exit_mode();
 667         }
 668 #endif
 669
 670 #if VCODEC_CLOCK_ENABLE
 671         if (pservice->pd_video)
 672                 clk_disable_unprepare(pservice->pd_video);
 673         if (pservice->hclk_vcodec)
 674                 clk_disable_unprepare(pservice->hclk_vcodec);
 675         if (pservice->aclk_vcodec)
 676                 clk_disable_unprepare(pservice->aclk_vcodec);
 677         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC) {
 678                 if (pservice->clk_core)
 679                         clk_disable_unprepare(pservice->clk_core);
 680                 if (pservice->clk_cabac)
 681                         clk_disable_unprepare(pservice->clk_cabac);
 682         }
 683 #endif
 684
 685         wake_unlock(&pservice->wake_lock);
 686         pr_info("done\n");
 687 }
 688
 689 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 690 {
 691         queue_delayed_work(system_nrt_wq, &pservice->power_off_work, VPU_POWER_OFF_DELAY);
 692 }
 693
 694 static void vpu_power_off_work(struct work_struct *work_s)
 695 {
 696         struct delayed_work *dlwork = container_of(work_s, struct delayed_work, work);
 697         struct vpu_service_info *pservice = container_of(dlwork, struct vpu_service_info, power_off_work);
 698
 699         if (mutex_trylock(&pservice->lock)) {
 700                 vpu_service_power_off(pservice);
 701                 mutex_unlock(&pservice->lock);
 702         } else {
 703                 /* Come back later if the device is busy... */
 704                 vpu_queue_power_off_work(pservice);
 705         }
 706 }
 707
 708 static void vpu_service_power_on(struct vpu_service_info *pservice)
 709 {
 710         static ktime_t last;
 711         ktime_t now = ktime_get();
 712         if (ktime_to_ns(ktime_sub(now, last)) > NSEC_PER_SEC) {
 713                 cancel_delayed_work_sync(&pservice->power_off_work);
 714                 vpu_queue_power_off_work(pservice);
 715                 last = now;
 716         }
 717         if (pservice->enabled)
 718                 return ;
 719
 720         pservice->enabled = true;
 721         printk("%s: power on\n", dev_name(pservice->dev));
 722
 723 #if VCODEC_CLOCK_ENABLE
 724         if (pservice->aclk_vcodec)
 725                 clk_prepare_enable(pservice->aclk_vcodec);
 726
 727         if (pservice->hclk_vcodec)
 728                 clk_prepare_enable(pservice->hclk_vcodec);
 729
 730         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC) {
 731                 if (pservice->clk_core)
 732                         clk_prepare_enable(pservice->clk_core);
 733         if (pservice->clk_cabac)
 734                 clk_prepare_enable(pservice->clk_cabac);
 735         }
 736
 737         if (pservice->pd_video)
 738                 clk_prepare_enable(pservice->pd_video);
 739 #endif
 740
 741 #if defined(CONFIG_ARCH_RK319X)
 742         /// select aclk_vepu as vcodec clock source.
 743 #define BIT_VCODEC_SEL  (1<<7)
 744         writel_relaxed(readl_relaxed(RK319X_GRF_BASE + GRF_SOC_CON1) |
 745                 (BIT_VCODEC_SEL) | (BIT_VCODEC_SEL << 16),
 746                 RK319X_GRF_BASE + GRF_SOC_CON1);
 747 #endif
 748 #define BIT_VCODEC_CLK_SEL      (1<<10)
 749         if (cpu_is_rk312x())
 750                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1) |
 751                         BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 752                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 753
 754         udelay(10);
 755         wake_lock(&pservice->wake_lock);
 756
 757 #if defined(CONFIG_VCODEC_MMU)
 758         if (pservice->mmu_dev)
 759                 rockchip_iovmm_activate(pservice->dev);
 760 #endif
 761 }
 762
 763 static inline bool reg_check_rmvb_wmv(vpu_reg *reg)
 764 {
 765         unsigned long type = (reg->reg[3] & 0xF0000000) >> 28;
 766         return ((type == 8) || (type == 4));
 767 }
 768
 769 static inline bool reg_check_interlace(vpu_reg *reg)
 770 {
 771         unsigned long type = (reg->reg[3] & (1 << 23));
 772         return (type > 0);
 773 }
 774
 775 static inline enum VPU_DEC_FMT reg_check_fmt(vpu_reg *reg)
 776 {
 777         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] & 0xF0000000) >> 28);
 778         return type;
 779 }
 780
 781 static inline int reg_probe_width(vpu_reg *reg)
 782 {
 783         int width_in_mb = reg->reg[4] >> 23;
 784         return width_in_mb * 16;
 785 }
 786
 787 #if defined(CONFIG_VCODEC_MMU)
 788 static int vcodec_bufid_to_iova(struct vpu_service_info *pservice, u8 *tbl,
 789                                 int size, vpu_reg *reg,
 790                                 struct extra_info_for_iommu *ext_inf)
 791 {
 792         int i;
 793         int usr_fd = 0;
 794         int offset = 0;
 795
 796         if (tbl == NULL || size <= 0) {
 797                 dev_err(pservice->dev, "input arguments invalidate\n");
 798                 return -1;
 799         }
 800
 801         vpu_service_power_on(pservice);
 802
 803         for (i = 0; i < size; i++) {
 804                 usr_fd = reg->reg[tbl[i]] & 0x3FF;
 805
 806                 if (tbl[i] == 41 && pservice->hw_info->hw_id != HEVC_ID &&
 807                     (reg->type == VPU_DEC || reg->type == VPU_DEC_PP))
 808                         /* special for vpu dec num 41 regitster */
 809                         offset = reg->reg[tbl[i]] >> 10 << 4;
 810                 else
 811                         offset = reg->reg[tbl[i]] >> 10;
 812
 813                 if (usr_fd != 0) {
 814                         struct ion_handle *hdl;
 815                         int ret = 0;
 816                         struct vcodec_mem_region *mem_region;
 817
 818                         hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
 819                         if (IS_ERR(hdl)) {
 820                                 dev_err(pservice->dev, "import dma-buf from fd %d failed, reg[%d]\n", usr_fd, tbl[i]);
 821                                 return PTR_ERR(hdl);
 822                         }
 823
 824                         mem_region = kzalloc(sizeof(struct vcodec_mem_region), GFP_KERNEL);
 825
 826                         if (mem_region == NULL) {
 827                                 dev_err(pservice->dev, "allocate memory for iommu memory region failed\n");
 828                                 ion_free(pservice->ion_client, hdl);
 829                                 return -1;
 830                         }
 831
 832                         mem_region->hdl = hdl;
 833                         vcodec_enter_mode(pservice->dev_id);
 834                         ret = ion_map_iommu(pservice->dev, pservice->ion_client, mem_region->hdl, &mem_region->iova, &mem_region->len);
 835                         vcodec_exit_mode();
 836
 837                         if (ret < 0) {
 838                                 dev_err(pservice->dev, "ion map iommu failed\n");
 839                                 kfree(mem_region);
 840                                 ion_free(pservice->ion_client, hdl);
 841                                 return ret;
 842                         }
 843                         reg->reg[tbl[i]] = mem_region->iova + offset;
 844                         INIT_LIST_HEAD(&mem_region->reg_lnk);
 845                         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 846                 }
 847         }
 848
 849         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
 850                 for (i=0; i<ext_inf->cnt; i++) {
 851                         pr_info("reg[%d] + offset %d\n", ext_inf->elem[i].index, ext_inf->elem[i].offset);
 852                         reg->reg[ext_inf->elem[i].index] += ext_inf->elem[i].offset;
 853                 }
 854         }
 855
 856         return 0;
 857 }
 858
 859 static int vcodec_reg_address_translate(struct vpu_service_info *pservice,
 860                                         vpu_reg *reg,
 861                                         struct extra_info_for_iommu *ext_inf)
 862 {
 863         VPU_HW_ID hw_id;
 864         u8 *tbl;
 865         int size = 0;
 866
 867         hw_id = pservice->hw_info->hw_id;
 868
 869         if (hw_id == HEVC_ID) {
 870                 tbl = addr_tbl_hevc_dec;
 871                 size = sizeof(addr_tbl_hevc_dec);
 872         } else {
 873                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
 874                         switch (reg_check_fmt(reg)) {
 875                         case VPU_DEC_FMT_H264:
 876                                 {
 877                                         tbl = addr_tbl_vpu_h264dec;
 878                                         size = sizeof(addr_tbl_vpu_h264dec);
 879                                         break;
 880                                 }
 881                         case VPU_DEC_FMT_VP8:
 882                         case VPU_DEC_FMT_VP7:
 883                                 {
 884                                         tbl = addr_tbl_vpu_vp8dec;
 885                                         size = sizeof(addr_tbl_vpu_vp8dec);
 886                                         break;
 887                                 }
 888
 889                         case VPU_DEC_FMT_VP6:
 890                                 {
 891                                         tbl = addr_tbl_vpu_vp6dec;
 892                                         size = sizeof(addr_tbl_vpu_vp6dec);
 893                                         break;
 894                                 }
 895                         case VPU_DEC_FMT_VC1:
 896                                 {
 897                                         tbl = addr_tbl_vpu_vc1dec;
 898                                         size = sizeof(addr_tbl_vpu_vc1dec);
 899                                         break;
 900                                 }
 901
 902                         case VPU_DEC_FMT_JPEG:
 903                                 {
 904                                         tbl = addr_tbl_vpu_jpegdec;
 905                                         size = sizeof(addr_tbl_vpu_jpegdec);
 906                                         break;
 907                                 }
 908                         default:
 909                                 tbl = addr_tbl_vpu_defaultdec;
 910                                 size = sizeof(addr_tbl_vpu_defaultdec);
 911                                 break;
 912                         }
 913                 } else if (reg->type == VPU_ENC) {
 914                         tbl = addr_tbl_vpu_enc;
 915                         size = sizeof(addr_tbl_vpu_enc);
 916                 }
 917         }
 918
 919         if (size != 0) {
 920                 return vcodec_bufid_to_iova(pservice, tbl, size, reg, ext_inf);
 921         } else {
 922                 return -1;
 923         }
 924 }
 925 #endif
 926
 927 static vpu_reg *reg_init(struct vpu_service_info *pservice, vpu_session *session, void __user *src, unsigned long size)
 928 {
 929         int extra_size = 0;
 930         struct extra_info_for_iommu extra_info;
 931         vpu_reg *reg = kmalloc(sizeof(vpu_reg)+pservice->reg_size, GFP_KERNEL);
 932         if (NULL == reg) {
 933                 pr_err("error: kmalloc fail in reg_init\n");
 934                 return NULL;
 935         }
 936
 937         if (size > pservice->reg_size) {
 938                 /*printk("warning: vpu reg size %lu is larger than hw reg size %lu\n", size, pservice->reg_size);
 939                 size = pservice->reg_size;*/
 940                 extra_size = size - pservice->reg_size;
 941                 size = pservice->reg_size;
 942         }
 943         reg->session = session;
 944         reg->type = session->type;
 945         reg->size = size;
 946         reg->freq = VPU_FREQ_DEFAULT;
 947         reg->reg = (unsigned long *)&reg[1];
 948         INIT_LIST_HEAD(&reg->session_link);
 949         INIT_LIST_HEAD(&reg->status_link);
 950
 951 #if defined(CONFIG_VCODEC_MMU)
 952         if (pservice->mmu_dev)
 953                 INIT_LIST_HEAD(&reg->mem_region_list);
 954 #endif
 955
 956         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
 957                 pr_err("error: copy_from_user failed in reg_init\n");
 958                 kfree(reg);
 959                 return NULL;
 960         }
 961
 962         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
 963                 pr_err("error: copy_from_user failed in reg_init\n");
 964                 kfree(reg);
 965                 return NULL;
 966         }
 967
 968 #if defined(CONFIG_VCODEC_MMU)
 969         if (pservice->mmu_dev && 0 > vcodec_reg_address_translate(pservice, reg, &extra_info)) {
 970                 pr_err("error: translate reg address failed\n");
 971                 kfree(reg);
 972                 return NULL;
 973         }
 974 #endif
 975
 976         mutex_lock(&pservice->lock);
 977         list_add_tail(&reg->status_link, &pservice->waiting);
 978         list_add_tail(&reg->session_link, &session->waiting);
 979         mutex_unlock(&pservice->lock);
 980
 981         if (pservice->auto_freq) {
 982                 if (!soc_is_rk2928g()) {
 983                         if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
 984                                 if (reg_check_rmvb_wmv(reg)) {
 985                                         reg->freq = VPU_FREQ_200M;
 986                                 } else if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
 987                                         if (reg_probe_width(reg) > 3200) {
 988                                                 // raise frequency for 4k avc.
 989                                                 reg->freq = VPU_FREQ_500M;
 990                                         }
 991                                 } else {
 992                                         if (reg_check_interlace(reg)) {
 993                                                 reg->freq = VPU_FREQ_400M;
 994                                         }
 995                                 }
 996                         }
 997                         if (reg->type == VPU_PP) {
 998                                 reg->freq = VPU_FREQ_400M;
 999                         }
1000                 }
1001         }
1002
1003         /*reg->reg[2] &= (~0x800000L);*/
1004
1005         return reg;
1006 }
1007
1008 static void reg_deinit(struct vpu_service_info *pservice, vpu_reg *reg)
1009 {
1010 #if defined(CONFIG_VCODEC_MMU)
1011         struct vcodec_mem_region *mem_region = NULL, *n;
1012 #endif
1013
1014         list_del_init(&reg->session_link);
1015         list_del_init(&reg->status_link);
1016         if (reg == pservice->reg_codec)
1017                 pservice->reg_codec = NULL;
1018         if (reg == pservice->reg_pproc)
1019                 pservice->reg_pproc = NULL;
1020
1021 #if defined(CONFIG_VCODEC_MMU)
1022         // release memory region attach to this registers table.
1023         if (pservice->mmu_dev) {
1024                 list_for_each_entry_safe(mem_region, n, &reg->mem_region_list, reg_lnk) {
1025                         /* do not unmap iommu manually,
1026                            unmap will proccess when memory release */
1027                         vcodec_enter_mode(pservice->dev_id);
1028                         /*ion_unmap_iommu(pservice->dev,
1029                                         pservice->ion_client,
1030                                         mem_region->hdl);*/
1031                         vcodec_exit_mode();
1032                         ion_free(pservice->ion_client, mem_region->hdl);
1033                         list_del_init(&mem_region->reg_lnk);
1034                         kfree(mem_region);
1035                 }
1036         }
1037 #endif
1038
1039         kfree(reg);
1040 }
1041
1042 static void reg_from_wait_to_run(struct vpu_service_info *pservice, vpu_reg *reg)
1043 {
1044         list_del_init(&reg->status_link);
1045         list_add_tail(&reg->status_link, &pservice->running);
1046
1047         list_del_init(&reg->session_link);
1048         list_add_tail(&reg->session_link, &reg->session->running);
1049 }
1050
1051 static void reg_copy_from_hw(vpu_reg *reg, volatile u32 *src, u32 count)
1052 {
1053         int i;
1054         u32 *dst = (u32 *)&reg->reg[0];
1055
1056         for (i = 0; i < count; i++)
1057                 *dst++ = *src++;
1058 }
1059
1060 static void reg_from_run_to_done(struct vpu_service_info *pservice, vpu_reg *reg)
1061 {
1062         int irq_reg = -1;
1063         list_del_init(&reg->status_link);
1064         list_add_tail(&reg->status_link, &pservice->done);
1065
1066         list_del_init(&reg->session_link);
1067         list_add_tail(&reg->session_link, &reg->session->done);
1068
1069         vcodec_enter_mode(pservice->dev_id);
1070         switch (reg->type) {
1071         case VPU_ENC : {
1072                 pservice->reg_codec = NULL;
1073                 reg_copy_from_hw(reg, pservice->enc_dev.hwregs, pservice->hw_info->enc_reg_num);
1074                 irq_reg = ENC_INTERRUPT_REGISTER;
1075                 break;
1076         }
1077         case VPU_DEC : {
1078                 int reg_len = pservice->hw_info->hw_id == HEVC_ID ? REG_NUM_HEVC_DEC : REG_NUM_9190_DEC;
1079                 pservice->reg_codec = NULL;
1080                 reg_copy_from_hw(reg, pservice->dec_dev.hwregs, reg_len);
1081                 irq_reg = DEC_INTERRUPT_REGISTER;
1082                 break;
1083         }
1084         case VPU_PP : {
1085                 pservice->reg_pproc = NULL;
1086                 reg_copy_from_hw(reg, pservice->dec_dev.hwregs + PP_INTERRUPT_REGISTER, REG_NUM_9190_PP);
1087                 pservice->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1088                 break;
1089         }
1090         case VPU_DEC_PP : {
1091                 pservice->reg_codec = NULL;
1092                 pservice->reg_pproc = NULL;
1093                 reg_copy_from_hw(reg, pservice->dec_dev.hwregs, REG_NUM_9190_DEC_PP);
1094                 pservice->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1095                 break;
1096         }
1097         default : {
1098                 pr_err("error: copy reg from hw with unknown type %d\n", reg->type);
1099                 break;
1100         }
1101         }
1102         vcodec_exit_mode();
1103
1104         if (irq_reg != -1) {
1105                 reg->reg[irq_reg] = pservice->irq_status;
1106         }
1107
1108         atomic_sub(1, &reg->session->task_running);
1109         atomic_sub(1, &pservice->total_running);
1110         wake_up(&reg->session->wait);
1111 }
1112
1113 static void vpu_service_set_freq(struct vpu_service_info *pservice, vpu_reg *reg)
1114 {
1115         VPU_FREQ curr = atomic_read(&pservice->freq_status);
1116         if (curr == reg->freq) {
1117                 return ;
1118         }
1119         atomic_set(&pservice->freq_status, reg->freq);
1120         switch (reg->freq) {
1121         case VPU_FREQ_200M : {
1122                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1123                 //printk("default: 200M\n");
1124         } break;
1125         case VPU_FREQ_266M : {
1126                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1127                 //printk("default: 266M\n");
1128         } break;
1129         case VPU_FREQ_300M : {
1130                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1131                 //printk("default: 300M\n");
1132         } break;
1133         case VPU_FREQ_400M : {
1134                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1135                 //printk("default: 400M\n");
1136         } break;
1137         case VPU_FREQ_500M : {
1138                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1139         } break;
1140         case VPU_FREQ_600M : {
1141                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1142         } break;
1143         default : {
1144                 if (soc_is_rk2928g()) {
1145                         clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1146                 } else {
1147                         clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1148                 }
1149                 //printk("default: 300M\n");
1150         } break;
1151         }
1152 }
1153
1154 #if HEVC_SIM_ENABLE
1155 static void simulate_start(struct vpu_service_info *pservice);
1156 #endif
1157 static void reg_copy_to_hw(struct vpu_service_info *pservice, vpu_reg *reg)
1158 {
1159         int i;
1160         u32 *src = (u32 *)&reg->reg[0];
1161         atomic_add(1, &pservice->total_running);
1162         atomic_add(1, &reg->session->task_running);
1163         if (pservice->auto_freq) {
1164                 vpu_service_set_freq(pservice, reg);
1165         }
1166
1167         vcodec_enter_mode(pservice->dev_id);
1168
1169         switch (reg->type) {
1170         case VPU_ENC : {
1171                 int enc_count = pservice->hw_info->enc_reg_num;
1172                 u32 *dst = (u32 *)pservice->enc_dev.hwregs;
1173
1174                 pservice->reg_codec = reg;
1175
1176                 dst[VPU_REG_EN_ENC] = src[VPU_REG_EN_ENC] & 0x6;
1177
1178                 for (i = 0; i < VPU_REG_EN_ENC; i++)
1179                         dst[i] = src[i];
1180
1181                 for (i = VPU_REG_EN_ENC + 1; i < enc_count; i++)
1182                         dst[i] = src[i];
1183
1184                 VEPU_CLEAR_CACHE(dst);
1185
1186                 dsb();
1187
1188                 dst[VPU_REG_ENC_GATE] = src[VPU_REG_ENC_GATE] | VPU_REG_ENC_GATE_BIT;
1189                 dst[VPU_REG_EN_ENC]   = src[VPU_REG_EN_ENC];
1190
1191 #if VPU_SERVICE_SHOW_TIME
1192                 do_gettimeofday(&enc_start);
1193 #endif
1194
1195         } break;
1196         case VPU_DEC : {
1197                 u32 *dst = (u32 *)pservice->dec_dev.hwregs;
1198
1199                 pservice->reg_codec = reg;
1200
1201                 if (pservice->hw_info->hw_id != HEVC_ID) {
1202                         for (i = REG_NUM_9190_DEC - 1; i > VPU_REG_DEC_GATE; i--)
1203                                 dst[i] = src[i];
1204                         VDPU_CLEAR_CACHE(dst);
1205                 } else {
1206                         for (i = REG_NUM_HEVC_DEC - 1; i > VPU_REG_EN_DEC; i--)
1207                                 dst[i] = src[i];
1208                         HEVC_CLEAR_CACHE(dst);
1209                 }
1210
1211                 dsb();
1212
1213                 if (pservice->hw_info->hw_id != HEVC_ID) {
1214                         dst[VPU_REG_DEC_GATE] = src[VPU_REG_DEC_GATE] | VPU_REG_DEC_GATE_BIT;
1215                         dst[VPU_REG_EN_DEC]   = src[VPU_REG_EN_DEC];
1216                 } else {
1217                         dst[VPU_REG_EN_DEC] = src[VPU_REG_EN_DEC];
1218                 }
1219
1220                 dsb();
1221                 dmb();
1222
1223 #if VPU_SERVICE_SHOW_TIME
1224                 do_gettimeofday(&dec_start);
1225 #endif
1226
1227         } break;
1228         case VPU_PP : {
1229                 u32 *dst = (u32 *)pservice->dec_dev.hwregs + PP_INTERRUPT_REGISTER;
1230                 pservice->reg_pproc = reg;
1231
1232                 dst[VPU_REG_PP_GATE] = src[VPU_REG_PP_GATE] | VPU_REG_PP_GATE_BIT;
1233
1234                 for (i = VPU_REG_PP_GATE + 1; i < REG_NUM_9190_PP; i++)
1235                         dst[i] = src[i];
1236
1237                 dsb();
1238
1239                 dst[VPU_REG_EN_PP] = src[VPU_REG_EN_PP];
1240
1241 #if VPU_SERVICE_SHOW_TIME
1242                 do_gettimeofday(&pp_start);
1243 #endif
1244
1245         } break;
1246         case VPU_DEC_PP : {
1247                 u32 *dst = (u32 *)pservice->dec_dev.hwregs;
1248                 pservice->reg_codec = reg;
1249                 pservice->reg_pproc = reg;
1250
1251                 VDPU_SOFT_RESET(dst);
1252                 VDPU_CLEAR_CACHE(dst);
1253
1254                 for (i = VPU_REG_EN_DEC_PP + 1; i < REG_NUM_9190_DEC_PP; i++)
1255                         dst[i] = src[i];
1256
1257                 dst[VPU_REG_EN_DEC_PP]   = src[VPU_REG_EN_DEC_PP] | 0x2;
1258                 dsb();
1259
1260                 dst[VPU_REG_DEC_PP_GATE] = src[VPU_REG_DEC_PP_GATE] | VPU_REG_PP_GATE_BIT;
1261                 dst[VPU_REG_DEC_GATE]    = src[VPU_REG_DEC_GATE]    | VPU_REG_DEC_GATE_BIT;
1262                 dst[VPU_REG_EN_DEC]      = src[VPU_REG_EN_DEC];
1263
1264 #if VPU_SERVICE_SHOW_TIME
1265                 do_gettimeofday(&dec_start);
1266 #endif
1267
1268         } break;
1269         default : {
1270                 pr_err("error: unsupport session type %d", reg->type);
1271                 atomic_sub(1, &pservice->total_running);
1272                 atomic_sub(1, &reg->session->task_running);
1273                 break;
1274         }
1275         }
1276
1277         vcodec_exit_mode();
1278
1279 #if HEVC_SIM_ENABLE
1280         if (pservice->hw_info->hw_id == HEVC_ID) {
1281                 simulate_start(pservice);
1282         }
1283 #endif
1284 }
1285
1286 static void try_set_reg(struct vpu_service_info *pservice)
1287 {
1288         // first get reg from reg list
1289         if (!list_empty(&pservice->waiting)) {
1290                 int can_set = 0;
1291                 vpu_reg *reg = list_entry(pservice->waiting.next, vpu_reg, status_link);
1292
1293                 vpu_service_power_on(pservice);
1294
1295                 switch (reg->type) {
1296                 case VPU_ENC : {
1297                         if ((NULL == pservice->reg_codec) &&  (NULL == pservice->reg_pproc))
1298                                 can_set = 1;
1299                 } break;
1300                 case VPU_DEC : {
1301                         if (NULL == pservice->reg_codec)
1302                                 can_set = 1;
1303                         if (pservice->auto_freq && (NULL != pservice->reg_pproc)) {
1304                                 can_set = 0;
1305                         }
1306                 } break;
1307                 case VPU_PP : {
1308                         if (NULL == pservice->reg_codec) {
1309                                 if (NULL == pservice->reg_pproc)
1310                                         can_set = 1;
1311                         } else {
1312                                 if ((VPU_DEC == pservice->reg_codec->type) && (NULL == pservice->reg_pproc))
1313                                         can_set = 1;
1314                                 // can not charge frequency when vpu is working
1315                                 if (pservice->auto_freq) {
1316                                         can_set = 0;
1317                                 }
1318                         }
1319                 } break;
1320                 case VPU_DEC_PP : {
1321                         if ((NULL == pservice->reg_codec) && (NULL == pservice->reg_pproc))
1322                                 can_set = 1;
1323                         } break;
1324                 default : {
1325                         printk("undefined reg type %d\n", reg->type);
1326                 } break;
1327                 }
1328                 if (can_set) {
1329                         reg_from_wait_to_run(pservice, reg);
1330                         reg_copy_to_hw(pservice, reg);
1331                 }
1332         }
1333 }
1334
1335 static int return_reg(struct vpu_service_info *pservice, vpu_reg *reg, u32 __user *dst)
1336 {
1337         int ret = 0;
1338         switch (reg->type) {
1339         case VPU_ENC : {
1340                 if (copy_to_user(dst, &reg->reg[0], pservice->hw_info->enc_io_size))
1341                         ret = -EFAULT;
1342                 break;
1343         }
1344         case VPU_DEC : {
1345                 int reg_len = pservice->hw_info->hw_id == HEVC_ID ? REG_NUM_HEVC_DEC : REG_NUM_9190_DEC;
1346                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(reg_len)))
1347                         ret = -EFAULT;
1348                 break;
1349         }
1350         case VPU_PP : {
1351                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_PP)))
1352                         ret = -EFAULT;
1353                 break;
1354         }
1355         case VPU_DEC_PP : {
1356                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_DEC_PP)))
1357                         ret = -EFAULT;
1358                 break;
1359         }
1360         default : {
1361                 ret = -EFAULT;
1362                 pr_err("error: copy reg to user with unknown type %d\n", reg->type);
1363                 break;
1364         }
1365         }
1366         reg_deinit(pservice, reg);
1367         return ret;
1368 }
1369
1370 static long vpu_service_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1371 {
1372     struct vpu_service_info *pservice = container_of(filp->f_dentry->d_inode->i_cdev, struct vpu_service_info, cdev);
1373         vpu_session *session = (vpu_session *)filp->private_data;
1374         if (NULL == session) {
1375                 return -EINVAL;
1376         }
1377
1378         switch (cmd) {
1379         case VPU_IOC_SET_CLIENT_TYPE : {
1380                 session->type = (VPU_CLIENT_TYPE)arg;
1381                 break;
1382         }
1383         case VPU_IOC_GET_HW_FUSE_STATUS : {
1384                 vpu_request req;
1385                 if (copy_from_user(&req, (void __user *)arg, sizeof(vpu_request))) {
1386                         pr_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_from_user failed\n");
1387                         return -EFAULT;
1388                 } else {
1389                         if (VPU_ENC != session->type) {
1390                                 if (copy_to_user((void __user *)req.req, &pservice->dec_config, sizeof(VPUHwDecConfig_t))) {
1391                                         pr_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n", session->type);
1392                                         return -EFAULT;
1393                                 }
1394                         } else {
1395                                 if (copy_to_user((void __user *)req.req, &pservice->enc_config, sizeof(VPUHwEncConfig_t))) {
1396                                         pr_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n", session->type);
1397                                         return -EFAULT;
1398                                 }
1399                         }
1400                 }
1401
1402                 break;
1403         }
1404         case VPU_IOC_SET_REG : {
1405                 vpu_request req;
1406                 vpu_reg *reg;
1407                 if (copy_from_user(&req, (void __user *)arg, sizeof(vpu_request))) {
1408                         pr_err("error: VPU_IOC_SET_REG copy_from_user failed\n");
1409                         return -EFAULT;
1410                 }
1411                 reg = reg_init(pservice, session, (void __user *)req.req, req.size);
1412                 if (NULL == reg) {
1413                         return -EFAULT;
1414                 } else {
1415                         mutex_lock(&pservice->lock);
1416                         try_set_reg(pservice);
1417                         mutex_unlock(&pservice->lock);
1418                 }
1419
1420                 break;
1421         }
1422         case VPU_IOC_GET_REG : {
1423                 vpu_request req;
1424                 vpu_reg *reg;
1425                 if (copy_from_user(&req, (void __user *)arg, sizeof(vpu_request))) {
1426                         pr_err("error: VPU_IOC_GET_REG copy_from_user failed\n");
1427                         return -EFAULT;
1428                 } else {
1429                         int ret = wait_event_timeout(session->wait, !list_empty(&session->done), VPU_TIMEOUT_DELAY);
1430                         if (!list_empty(&session->done)) {
1431                                 if (ret < 0) {
1432                                         pr_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session->pid, ret);
1433                                 }
1434                                 ret = 0;
1435                         } else {
1436                                 if (unlikely(ret < 0)) {
1437                                         pr_err("error: pid %d wait task ret %d\n", session->pid, ret);
1438                                 } else if (0 == ret) {
1439                                         pr_err("error: pid %d wait %d task done timeout\n", session->pid, atomic_read(&session->task_running));
1440                                         ret = -ETIMEDOUT;
1441                                 }
1442                         }
1443                         if (ret < 0) {
1444                                 int task_running = atomic_read(&session->task_running);
1445                                 mutex_lock(&pservice->lock);
1446                                 vpu_service_dump(pservice);
1447                                 if (task_running) {
1448                                         atomic_set(&session->task_running, 0);
1449                                         atomic_sub(task_running, &pservice->total_running);
1450                                         printk("%d task is running but not return, reset hardware...", task_running);
1451                                         vpu_reset(pservice);
1452                                         printk("done\n");
1453                                 }
1454                                 vpu_service_session_clear(pservice, session);
1455                                 mutex_unlock(&pservice->lock);
1456                                 return ret;
1457                         }
1458                 }
1459                 mutex_lock(&pservice->lock);
1460                 reg = list_entry(session->done.next, vpu_reg, session_link);
1461                 return_reg(pservice, reg, (u32 __user *)req.req);
1462                 mutex_unlock(&pservice->lock);
1463                 break;
1464         }
1465         case VPU_IOC_PROBE_IOMMU_STATUS: {
1466                 int iommu_enable = 0;
1467
1468 #if defined(CONFIG_VCODEC_MMU)
1469                 iommu_enable = pservice->mmu_dev ? 1 : 0;
1470 #endif
1471
1472                 if (copy_to_user((void __user *)arg, &iommu_enable, sizeof(int))) {
1473                         pr_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1474                         return -EFAULT;
1475                 }
1476                 break;
1477         }
1478         default : {
1479                 pr_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1480                 break;
1481         }
1482         }
1483
1484         return 0;
1485 }
1486
1487 static int vpu_service_check_hw(vpu_service_info *p, unsigned long hw_addr)
1488 {
1489         int ret = -EINVAL, i = 0;
1490         volatile u32 *tmp = (volatile u32 *)ioremap_nocache(hw_addr, 0x4);
1491         u32 enc_id = *tmp;
1492
1493 #if HEVC_SIM_ENABLE
1494         /// temporary, hevc driver test.
1495         if (strncmp(dev_name(p->dev), "hevc_service", strlen("hevc_service")) == 0) {
1496                 p->hw_info = &vpu_hw_set[2];
1497                 return 0;
1498         }
1499 #endif
1500         enc_id = (enc_id >> 16) & 0xFFFF;
1501         pr_info("checking hw id %x\n", enc_id);
1502         p->hw_info = NULL;
1503         for (i = 0; i < ARRAY_SIZE(vpu_hw_set); i++) {
1504                 if (enc_id == vpu_hw_set[i].hw_id) {
1505                         p->hw_info = &vpu_hw_set[i];
1506                         ret = 0;
1507                         break;
1508                 }
1509         }
1510         iounmap((void *)tmp);
1511         return ret;
1512 }
1513
1514 static int vpu_service_open(struct inode *inode, struct file *filp)
1515 {
1516         struct vpu_service_info *pservice = container_of(inode->i_cdev, struct vpu_service_info, cdev);
1517         vpu_session *session = (vpu_session *)kmalloc(sizeof(vpu_session), GFP_KERNEL);
1518         if (NULL == session) {
1519                 pr_err("error: unable to allocate memory for vpu_session.");
1520                 return -ENOMEM;
1521         }
1522
1523         session->type   = VPU_TYPE_BUTT;
1524         session->pid    = current->pid;
1525         INIT_LIST_HEAD(&session->waiting);
1526         INIT_LIST_HEAD(&session->running);
1527         INIT_LIST_HEAD(&session->done);
1528         INIT_LIST_HEAD(&session->list_session);
1529         init_waitqueue_head(&session->wait);
1530         atomic_set(&session->task_running, 0);
1531         mutex_lock(&pservice->lock);
1532         list_add_tail(&session->list_session, &pservice->session);
1533         filp->private_data = (void *)session;
1534         mutex_unlock(&pservice->lock);
1535
1536         pr_debug("dev opened\n");
1537         return nonseekable_open(inode, filp);
1538 }
1539
1540 static int vpu_service_release(struct inode *inode, struct file *filp)
1541 {
1542         struct vpu_service_info *pservice = container_of(inode->i_cdev, struct vpu_service_info, cdev);
1543         int task_running;
1544         vpu_session *session = (vpu_session *)filp->private_data;
1545         if (NULL == session)
1546                 return -EINVAL;
1547
1548         task_running = atomic_read(&session->task_running);
1549         if (task_running) {
1550                 pr_err("error: vpu_service session %d still has %d task running when closing\n", session->pid, task_running);
1551                 msleep(50);
1552         }
1553         wake_up(&session->wait);
1554
1555         mutex_lock(&pservice->lock);
1556         /* remove this filp from the asynchronusly notified filp's */
1557         list_del_init(&session->list_session);
1558         vpu_service_session_clear(pservice, session);
1559         kfree(session);
1560         filp->private_data = NULL;
1561         mutex_unlock(&pservice->lock);
1562
1563         pr_debug("dev closed\n");
1564         return 0;
1565 }
1566
1567 static const struct file_operations vpu_service_fops = {
1568         .unlocked_ioctl = vpu_service_ioctl,
1569         .open           = vpu_service_open,
1570         .release        = vpu_service_release,
1571         //.fasync       = vpu_service_fasync,
1572 };
1573
1574 static irqreturn_t vdpu_irq(int irq, void *dev_id);
1575 static irqreturn_t vdpu_isr(int irq, void *dev_id);
1576 static irqreturn_t vepu_irq(int irq, void *dev_id);
1577 static irqreturn_t vepu_isr(int irq, void *dev_id);
1578 static void get_hw_info(struct vpu_service_info *pservice);
1579
1580 #if HEVC_SIM_ENABLE
1581 static void simulate_work(struct work_struct *work_s)
1582 {
1583         struct delayed_work *dlwork = container_of(work_s, struct delayed_work, work);
1584         struct vpu_service_info *pservice = container_of(dlwork, struct vpu_service_info, simulate_work);
1585         vpu_device *dev = &pservice->dec_dev;
1586
1587         if (!list_empty(&pservice->running)) {
1588                 atomic_add(1, &dev->irq_count_codec);
1589                 vdpu_isr(0, (void*)pservice);
1590         } else {
1591                 //simulate_start(pservice);
1592                 pr_err("empty running queue\n");
1593         }
1594 }
1595
1596 static void simulate_init(struct vpu_service_info *pservice)
1597 {
1598         INIT_DELAYED_WORK(&pservice->simulate_work, simulate_work);
1599 }
1600
1601 static void simulate_start(struct vpu_service_info *pservice)
1602 {
1603         cancel_delayed_work_sync(&pservice->power_off_work);
1604         queue_delayed_work(system_nrt_wq, &pservice->simulate_work, VPU_SIMULATE_DELAY);
1605 }
1606 #endif
1607
1608 #ifdef CONFIG_VCODEC_MMU
1609 static struct device *rockchip_get_sysmmu_device_by_compatible(const char *compt)
1610 {
1611         struct device_node *dn = NULL;
1612         struct platform_device *pd = NULL;
1613         struct device *ret = NULL ;
1614
1615         dn = of_find_compatible_node(NULL,NULL,compt);
1616         if(!dn) {
1617                 printk("can't find device node %s \r\n",compt);
1618                 return NULL;
1619         }
1620
1621         pd = of_find_device_by_node(dn);
1622         if(!pd) {
1623                 printk("can't find platform device in device node %s\n",compt);
1624                 return  NULL;
1625         }
1626         ret = &pd->dev;
1627
1628         return ret;
1629
1630 }
1631 #ifdef CONFIG_IOMMU_API
1632 static inline void platform_set_sysmmu(struct device *iommu,
1633         struct device *dev)
1634 {
1635         dev->archdata.iommu = iommu;
1636 }
1637 #else
1638 static inline void platform_set_sysmmu(struct device *iommu,
1639         struct device *dev)
1640 {
1641 }
1642 #endif
1643
1644 int vcodec_sysmmu_fault_handler(struct device *dev,
1645                                 enum rk_iommu_inttype itype,
1646                                 unsigned long pgtable_base,
1647                                 unsigned long fault_addr, unsigned int status)
1648 {
1649         struct platform_device *pdev;
1650         struct vpu_service_info *pservice;
1651
1652         pr_info("%s in\n", __func__);
1653
1654         pdev = container_of(dev, struct platform_device, dev);
1655
1656         pservice = platform_get_drvdata(pdev);
1657
1658         if (pservice->reg_codec) {
1659                 struct vcodec_mem_region *mem, *n;
1660                 int i = 0;
1661                 pr_info("vcodec, fault addr 0x%08x\n", (u32)fault_addr);
1662                 list_for_each_entry_safe(mem, n,
1663                                          &pservice->reg_codec->mem_region_list,
1664                                          reg_lnk) {
1665                         pr_info("vcodec, mem region [%02d] 0x%08x %ld\n",
1666                                 i, (u32)mem->iova, mem->len);
1667                         i++;
1668                 }
1669
1670                 pr_alert("vcodec, page fault occur, reset hw\n");
1671                 pservice->reg_codec->reg[101] = 1;
1672                 vpu_reset(pservice);
1673         }
1674
1675         return 0;
1676 }
1677 #endif
1678
1679 #if HEVC_TEST_ENABLE
1680 static int hevc_test_case0(vpu_service_info *pservice);
1681 #endif
1682 #if defined(CONFIG_ION_ROCKCHIP)
1683 extern struct ion_client *rockchip_ion_client_create(const char * name);
1684 #endif
1685 static int vcodec_probe(struct platform_device *pdev)
1686 {
1687         int ret = 0;
1688         struct resource *res = NULL;
1689         struct device *dev = &pdev->dev;
1690         void __iomem *regs = NULL;
1691         struct device_node *np = pdev->dev.of_node;
1692         struct vpu_service_info *pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
1693         char *prop = (char*)dev_name(dev);
1694 #if defined(CONFIG_VCODEC_MMU)
1695         u32 iommu_en = 0;
1696         char mmu_dev_dts_name[40];
1697         of_property_read_u32(np, "iommu_enabled", &iommu_en);
1698 #endif
1699
1700         pr_info("probe device %s\n", dev_name(dev));
1701
1702         of_property_read_string(np, "name", (const char**)&prop);
1703         dev_set_name(dev, prop);
1704
1705         if (strcmp(dev_name(dev), "hevc_service") == 0) {
1706                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
1707         } else if (strcmp(dev_name(dev), "vpu_service") == 0) {
1708                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
1709         } else {
1710                 dev_err(dev, "Unknown device %s to probe\n", dev_name(dev));
1711                 return -1;
1712         }
1713
1714         mutex_init(&g_mode_mutex);
1715         vcodec_enter_mode(pservice->dev_id);
1716
1717         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
1718         INIT_LIST_HEAD(&pservice->waiting);
1719         INIT_LIST_HEAD(&pservice->running);
1720         INIT_LIST_HEAD(&pservice->done);
1721         INIT_LIST_HEAD(&pservice->session);
1722         mutex_init(&pservice->lock);
1723         pservice->reg_codec     = NULL;
1724         pservice->reg_pproc     = NULL;
1725         atomic_set(&pservice->total_running, 0);
1726         pservice->enabled = false;
1727 #if defined(CONFIG_VCODEC_MMU)
1728         pservice->mmu_dev = NULL;
1729 #endif
1730         pservice->dev = dev;
1731
1732         if (0 > vpu_get_clk(pservice))
1733                 goto err;
1734
1735         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
1736
1737         vpu_service_power_on(pservice);
1738
1739         mdelay(1);
1740
1741         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1742
1743         res->flags &= ~IORESOURCE_CACHEABLE;
1744
1745         regs = devm_ioremap_resource(pservice->dev, res);
1746         if (IS_ERR(regs)) {
1747                 ret = PTR_ERR(regs);
1748                 goto err;
1749         }
1750
1751         {
1752                 u32 offset = res->start;
1753                 if (cpu_is_rk3036()) {
1754                         if (pservice->dev_id == VCODEC_DEVICE_ID_VPU)
1755                                 offset += 0x400;
1756                 }
1757                 ret = vpu_service_check_hw(pservice, offset);
1758                 if (ret < 0) {
1759                         pr_err("error: hw info check faild\n");
1760                         goto err;
1761                 }
1762         }
1763
1764         /// define regs address.
1765         pservice->dec_dev.iobaseaddr = res->start + pservice->hw_info->dec_offset;
1766         pservice->dec_dev.iosize     = pservice->hw_info->dec_io_size;
1767
1768         pservice->dec_dev.hwregs = (volatile u32 *)((u8 *)regs + pservice->hw_info->dec_offset);
1769
1770         pservice->reg_size   = pservice->dec_dev.iosize;
1771
1772         if (pservice->hw_info->hw_id != HEVC_ID && !cpu_is_rk3036()) {
1773                 pservice->enc_dev.iobaseaddr = res->start + pservice->hw_info->enc_offset;
1774                 pservice->enc_dev.iosize     = pservice->hw_info->enc_io_size;
1775
1776                 pservice->reg_size = pservice->reg_size > pservice->enc_dev.iosize ? pservice->reg_size : pservice->enc_dev.iosize;
1777
1778                 pservice->enc_dev.hwregs = (volatile u32 *)((u8 *)regs + pservice->hw_info->enc_offset);
1779
1780                 pservice->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
1781                 if (pservice->irq_enc < 0) {
1782                         dev_err(pservice->dev, "cannot find IRQ encoder\n");
1783                         ret = -ENXIO;
1784                         goto err;
1785                 }
1786
1787                 ret = devm_request_threaded_irq(pservice->dev, pservice->irq_enc, vepu_irq, vepu_isr, 0, dev_name(pservice->dev), (void *)pservice);
1788                 if (ret) {
1789                         dev_err(pservice->dev, "error: can't request vepu irq %d\n", pservice->irq_enc);
1790                         goto err;
1791                 }
1792         }
1793
1794         pservice->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
1795         if (pservice->irq_dec < 0) {
1796                 dev_err(pservice->dev, "cannot find IRQ decoder\n");
1797                 ret = -ENXIO;
1798                 goto err;
1799         }
1800
1801         /* get the IRQ line */
1802         ret = devm_request_threaded_irq(pservice->dev, pservice->irq_dec, vdpu_irq, vdpu_isr, 0, dev_name(pservice->dev), (void *)pservice);
1803         if (ret) {
1804                 dev_err(pservice->dev, "error: can't request vdpu irq %d\n", pservice->irq_dec);
1805                 goto err;
1806         }
1807
1808         atomic_set(&pservice->dec_dev.irq_count_codec, 0);
1809         atomic_set(&pservice->dec_dev.irq_count_pp, 0);
1810         atomic_set(&pservice->enc_dev.irq_count_codec, 0);
1811         atomic_set(&pservice->enc_dev.irq_count_pp, 0);
1812
1813         /// create device
1814         ret = alloc_chrdev_region(&pservice->dev_t, 0, 1, dev_name(dev));
1815         if (ret) {
1816                 dev_err(dev, "alloc dev_t failed\n");
1817                 goto err;
1818         }
1819
1820         cdev_init(&pservice->cdev, &vpu_service_fops);
1821
1822         pservice->cdev.owner = THIS_MODULE;
1823         pservice->cdev.ops = &vpu_service_fops;
1824
1825         ret = cdev_add(&pservice->cdev, pservice->dev_t, 1);
1826
1827         if (ret) {
1828                 dev_err(dev, "add dev_t failed\n");
1829                 goto err;
1830         }
1831
1832         pservice->cls = class_create(THIS_MODULE, dev_name(dev));
1833
1834         if (IS_ERR(pservice->cls)) {
1835                 ret = PTR_ERR(pservice->cls);
1836                 dev_err(dev, "class_create err:%d\n", ret);
1837                 goto err;
1838         }
1839
1840         pservice->child_dev = device_create(pservice->cls, dev, pservice->dev_t, NULL, dev_name(dev));
1841
1842         platform_set_drvdata(pdev, pservice);
1843
1844         get_hw_info(pservice);
1845
1846
1847 #ifdef CONFIG_DEBUG_FS
1848         pservice->debugfs_dir = vcodec_debugfs_create_device_dir((char*)dev_name(dev), parent);
1849         if (pservice->debugfs_dir == NULL)
1850                 pr_err("create debugfs dir %s failed\n", dev_name(dev));
1851
1852         pservice->debugfs_file_regs =
1853                 debugfs_create_file("regs", 0664,
1854                                     pservice->debugfs_dir, pservice,
1855                                     &debug_vcodec_fops);
1856 #endif
1857
1858 #if defined(CONFIG_VCODEC_MMU)
1859         if (iommu_en) {
1860                 pservice->ion_client = rockchip_ion_client_create("vpu");
1861                 if (IS_ERR(pservice->ion_client)) {
1862                         dev_err(&pdev->dev, "failed to create ion client for vcodec");
1863                         return PTR_ERR(pservice->ion_client);
1864                 } else {
1865                         dev_info(&pdev->dev, "vcodec ion client create success!\n");
1866                 }
1867
1868                 if (pservice->hw_info->hw_id == HEVC_ID)
1869                         sprintf(mmu_dev_dts_name, HEVC_IOMMU_COMPATIBLE_NAME);
1870                 else
1871                         sprintf(mmu_dev_dts_name, VPU_IOMMU_COMPATIBLE_NAME);
1872
1873                 pservice->mmu_dev = rockchip_get_sysmmu_device_by_compatible(mmu_dev_dts_name);
1874
1875                 if (pservice->mmu_dev) {
1876                         platform_set_sysmmu(pservice->mmu_dev, pservice->dev);
1877                         rockchip_iovmm_activate(pservice->dev);
1878                 }
1879
1880                 rockchip_iovmm_set_fault_handler(pservice->dev, vcodec_sysmmu_fault_handler);
1881         }
1882 #endif
1883         vcodec_exit_mode();
1884         vpu_service_power_off(pservice);
1885
1886         pr_info("init success\n");
1887
1888 #if HEVC_SIM_ENABLE
1889         if (pservice->hw_info->hw_id == HEVC_ID)
1890                 simulate_init(pservice);
1891 #endif
1892
1893 #if HEVC_TEST_ENABLE
1894         hevc_test_case0(pservice);
1895 #endif
1896
1897         return 0;
1898
1899 err:
1900         pr_info("init failed\n");
1901         vcodec_exit_mode();
1902         vpu_service_power_off(pservice);
1903         vpu_put_clk(pservice);
1904         wake_lock_destroy(&pservice->wake_lock);
1905
1906         if (res)
1907                 devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
1908         if (pservice->irq_enc > 0)
1909                 free_irq(pservice->irq_enc, (void *)pservice);
1910         if (pservice->irq_dec > 0)
1911                 free_irq(pservice->irq_dec, (void *)pservice);
1912
1913         if (pservice->child_dev) {
1914                 device_destroy(pservice->cls, pservice->dev_t);
1915                 cdev_del(&pservice->cdev);
1916                 unregister_chrdev_region(pservice->dev_t, 1);
1917         }
1918
1919         if (pservice->cls)
1920                 class_destroy(pservice->cls);
1921
1922         return ret;
1923 }
1924
1925 static int vcodec_remove(struct platform_device *pdev)
1926 {
1927         struct vpu_service_info *pservice = platform_get_drvdata(pdev);
1928         struct resource *res;
1929
1930         device_destroy(pservice->cls, pservice->dev_t);
1931         class_destroy(pservice->cls);
1932         cdev_del(&pservice->cdev);
1933         unregister_chrdev_region(pservice->dev_t, 1);
1934
1935         free_irq(pservice->irq_enc, (void *)&pservice->enc_dev);
1936         free_irq(pservice->irq_dec, (void *)&pservice->dec_dev);
1937         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1938         devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
1939         vpu_put_clk(pservice);
1940         wake_lock_destroy(&pservice->wake_lock);
1941
1942 #ifdef CONFIG_DEBUG_FS
1943         debugfs_remove(pservice->debugfs_file_regs);
1944         debugfs_remove(pservice->debugfs_dir);
1945 #endif
1946
1947         return 0;
1948 }
1949
1950 #if defined(CONFIG_OF)
1951 static const struct of_device_id vcodec_service_dt_ids[] = {
1952         {.compatible = "vpu_service",},
1953         {.compatible = "rockchip,hevc_service",},
1954         {},
1955 };
1956 #endif
1957
1958 static struct platform_driver vcodec_driver = {
1959         .probe = vcodec_probe,
1960         .remove = vcodec_remove,
1961         .driver = {
1962                 .name = "vcodec",
1963                 .owner = THIS_MODULE,
1964 #if defined(CONFIG_OF)
1965                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
1966 #endif
1967         },
1968 };
1969
1970 static void get_hw_info(struct vpu_service_info *pservice)
1971 {
1972         VPUHwDecConfig_t *dec = &pservice->dec_config;
1973         VPUHwEncConfig_t *enc = &pservice->enc_config;
1974
1975         if (pservice->dev_id == VCODEC_DEVICE_ID_VPU) {
1976                 u32 configReg   = pservice->dec_dev.hwregs[VPU_DEC_HWCFG0];
1977                 u32 asicID      = pservice->dec_dev.hwregs[0];
1978
1979                 dec->h264Support    = (configReg >> DWL_H264_E) & 0x3U;
1980                 dec->jpegSupport    = (configReg >> DWL_JPEG_E) & 0x01U;
1981                 if (dec->jpegSupport && ((configReg >> DWL_PJPEG_E) & 0x01U))
1982                         dec->jpegSupport = JPEG_PROGRESSIVE;
1983                 dec->mpeg4Support   = (configReg >> DWL_MPEG4_E) & 0x3U;
1984                 dec->vc1Support     = (configReg >> DWL_VC1_E) & 0x3U;
1985                 dec->mpeg2Support   = (configReg >> DWL_MPEG2_E) & 0x01U;
1986                 dec->sorensonSparkSupport = (configReg >> DWL_SORENSONSPARK_E) & 0x01U;
1987                 dec->refBufSupport  = (configReg >> DWL_REF_BUFF_E) & 0x01U;
1988                 dec->vp6Support     = (configReg >> DWL_VP6_E) & 0x01U;
1989
1990                 if (soc_is_rk3190() || soc_is_rk3288())
1991                         dec->maxDecPicWidth = 4096;
1992                 else if (cpu_is_rk3036() || cpu_is_rk312x())
1993                         dec->maxDecPicWidth = 1920;
1994                 else
1995                         dec->maxDecPicWidth = configReg & 0x07FFU;
1996
1997                 /* 2nd Config register */
1998                 configReg   = pservice->dec_dev.hwregs[VPU_DEC_HWCFG1];
1999                 if (dec->refBufSupport) {
2000                         if ((configReg >> DWL_REF_BUFF_ILACE_E) & 0x01U)
2001                                 dec->refBufSupport |= 2;
2002                         if ((configReg >> DWL_REF_BUFF_DOUBLE_E) & 0x01U)
2003                                 dec->refBufSupport |= 4;
2004                 }
2005                 dec->customMpeg4Support = (configReg >> DWL_MPEG4_CUSTOM_E) & 0x01U;
2006                 dec->vp7Support     = (configReg >> DWL_VP7_E) & 0x01U;
2007                 dec->vp8Support     = (configReg >> DWL_VP8_E) & 0x01U;
2008                 dec->avsSupport     = (configReg >> DWL_AVS_E) & 0x01U;
2009
2010                 /* JPEG xtensions */
2011                 if (((asicID >> 16) >= 0x8190U) || ((asicID >> 16) == 0x6731U))
2012                         dec->jpegESupport = (configReg >> DWL_JPEG_EXT_E) & 0x01U;
2013                 else
2014                         dec->jpegESupport = JPEG_EXT_NOT_SUPPORTED;
2015
2016                 if (((asicID >> 16) >= 0x9170U) || ((asicID >> 16) == 0x6731U) )
2017                         dec->rvSupport = (configReg >> DWL_RV_E) & 0x03U;
2018                 else
2019                         dec->rvSupport = RV_NOT_SUPPORTED;
2020                 dec->mvcSupport = (configReg >> DWL_MVC_E) & 0x03U;
2021
2022                 if (dec->refBufSupport && (asicID >> 16) == 0x6731U )
2023                         dec->refBufSupport |= 8; /* enable HW support for offset */
2024
2025                 /// invalidate fuse register value in rk319x vpu and following.
2026                 if (!soc_is_rk3190() && !soc_is_rk3288() && !cpu_is_rk3036() && !cpu_is_rk312x()) {
2027                         VPUHwFuseStatus_t hwFuseSts;
2028                         /* Decoder fuse configuration */
2029                         u32 fuseReg = pservice->dec_dev.hwregs[VPU_DEC_HW_FUSE_CFG];
2030
2031                         hwFuseSts.h264SupportFuse = (fuseReg >> DWL_H264_FUSE_E) & 0x01U;
2032                         hwFuseSts.mpeg4SupportFuse = (fuseReg >> DWL_MPEG4_FUSE_E) & 0x01U;
2033                         hwFuseSts.mpeg2SupportFuse = (fuseReg >> DWL_MPEG2_FUSE_E) & 0x01U;
2034                         hwFuseSts.sorensonSparkSupportFuse = (fuseReg >> DWL_SORENSONSPARK_FUSE_E) & 0x01U;
2035                         hwFuseSts.jpegSupportFuse = (fuseReg >> DWL_JPEG_FUSE_E) & 0x01U;
2036                         hwFuseSts.vp6SupportFuse = (fuseReg >> DWL_VP6_FUSE_E) & 0x01U;
2037                         hwFuseSts.vc1SupportFuse = (fuseReg >> DWL_VC1_FUSE_E) & 0x01U;
2038                         hwFuseSts.jpegProgSupportFuse = (fuseReg >> DWL_PJPEG_FUSE_E) & 0x01U;
2039                         hwFuseSts.rvSupportFuse = (fuseReg >> DWL_RV_FUSE_E) & 0x01U;
2040                         hwFuseSts.avsSupportFuse = (fuseReg >> DWL_AVS_FUSE_E) & 0x01U;
2041                         hwFuseSts.vp7SupportFuse = (fuseReg >> DWL_VP7_FUSE_E) & 0x01U;
2042                         hwFuseSts.vp8SupportFuse = (fuseReg >> DWL_VP8_FUSE_E) & 0x01U;
2043                         hwFuseSts.customMpeg4SupportFuse = (fuseReg >> DWL_CUSTOM_MPEG4_FUSE_E) & 0x01U;
2044                         hwFuseSts.mvcSupportFuse = (fuseReg >> DWL_MVC_FUSE_E) & 0x01U;
2045
2046                         /* check max. decoder output width */
2047
2048                         if (fuseReg & 0x8000U)
2049                                 hwFuseSts.maxDecPicWidthFuse = 1920;
2050                         else if (fuseReg & 0x4000U)
2051                                 hwFuseSts.maxDecPicWidthFuse = 1280;
2052                         else if (fuseReg & 0x2000U)
2053                                 hwFuseSts.maxDecPicWidthFuse = 720;
2054                         else if (fuseReg & 0x1000U)
2055                                 hwFuseSts.maxDecPicWidthFuse = 352;
2056                         else    /* remove warning */
2057                                 hwFuseSts.maxDecPicWidthFuse = 352;
2058
2059                         hwFuseSts.refBufSupportFuse = (fuseReg >> DWL_REF_BUFF_FUSE_E) & 0x01U;
2060
2061                         /* Pp configuration */
2062                         configReg = pservice->dec_dev.hwregs[VPU_PP_HW_SYNTH_CFG];
2063
2064                         if ((configReg >> DWL_PP_E) & 0x01U) {
2065                                 dec->ppSupport = 1;
2066                                 dec->maxPpOutPicWidth = configReg & 0x07FFU;
2067                                 /*pHwCfg->ppConfig = (configReg >> DWL_CFG_E) & 0x0FU; */
2068                                 dec->ppConfig = configReg;
2069                         } else {
2070                                 dec->ppSupport = 0;
2071                                 dec->maxPpOutPicWidth = 0;
2072                                 dec->ppConfig = 0;
2073                         }
2074
2075                         /* check the HW versio */
2076                         if (((asicID >> 16) >= 0x8190U) || ((asicID >> 16) == 0x6731U)) {
2077                                 /* Pp configuration */
2078                                 configReg = pservice->dec_dev.hwregs[VPU_DEC_HW_FUSE_CFG];
2079                                 if ((configReg >> DWL_PP_E) & 0x01U) {
2080                                         /* Pp fuse configuration */
2081                                         u32 fuseRegPp = pservice->dec_dev.hwregs[VPU_PP_HW_FUSE_CFG];
2082
2083                                         if ((fuseRegPp >> DWL_PP_FUSE_E) & 0x01U) {
2084                                                 hwFuseSts.ppSupportFuse = 1;
2085                                                 /* check max. pp output width */
2086                                                 if (fuseRegPp & 0x8000U)
2087                                                         hwFuseSts.maxPpOutPicWidthFuse = 1920;
2088                                                 else if (fuseRegPp & 0x4000U)
2089                                                         hwFuseSts.maxPpOutPicWidthFuse = 1280;
2090                                                 else if (fuseRegPp & 0x2000U)
2091                                                         hwFuseSts.maxPpOutPicWidthFuse = 720;
2092                                                 else if (fuseRegPp & 0x1000U)
2093                                                         hwFuseSts.maxPpOutPicWidthFuse = 352;
2094                                                 else
2095                                                         hwFuseSts.maxPpOutPicWidthFuse = 352;
2096                                                 hwFuseSts.ppConfigFuse = fuseRegPp;
2097                                         } else {
2098                                                 hwFuseSts.ppSupportFuse = 0;
2099                                                 hwFuseSts.maxPpOutPicWidthFuse = 0;
2100                                                 hwFuseSts.ppConfigFuse = 0;
2101                                         }
2102                                 } else {
2103                                         hwFuseSts.ppSupportFuse = 0;
2104                                         hwFuseSts.maxPpOutPicWidthFuse = 0;
2105                                         hwFuseSts.ppConfigFuse = 0;
2106                                 }
2107
2108                                 if (dec->maxDecPicWidth > hwFuseSts.maxDecPicWidthFuse)
2109                                         dec->maxDecPicWidth = hwFuseSts.maxDecPicWidthFuse;
2110                                 if (dec->maxPpOutPicWidth > hwFuseSts.maxPpOutPicWidthFuse)
2111                                         dec->maxPpOutPicWidth = hwFuseSts.maxPpOutPicWidthFuse;
2112                                 if (!hwFuseSts.h264SupportFuse) dec->h264Support = H264_NOT_SUPPORTED;
2113                                 if (!hwFuseSts.mpeg4SupportFuse) dec->mpeg4Support = MPEG4_NOT_SUPPORTED;
2114                                 if (!hwFuseSts.customMpeg4SupportFuse) dec->customMpeg4Support = MPEG4_CUSTOM_NOT_SUPPORTED;
2115                                 if (!hwFuseSts.jpegSupportFuse) dec->jpegSupport = JPEG_NOT_SUPPORTED;
2116                                 if ((dec->jpegSupport == JPEG_PROGRESSIVE) && !hwFuseSts.jpegProgSupportFuse)
2117                                         dec->jpegSupport = JPEG_BASELINE;
2118                                 if (!hwFuseSts.mpeg2SupportFuse) dec->mpeg2Support = MPEG2_NOT_SUPPORTED;
2119                                 if (!hwFuseSts.vc1SupportFuse) dec->vc1Support = VC1_NOT_SUPPORTED;
2120                                 if (!hwFuseSts.vp6SupportFuse) dec->vp6Support = VP6_NOT_SUPPORTED;
2121                                 if (!hwFuseSts.vp7SupportFuse) dec->vp7Support = VP7_NOT_SUPPORTED;
2122                                 if (!hwFuseSts.vp8SupportFuse) dec->vp8Support = VP8_NOT_SUPPORTED;
2123                                 if (!hwFuseSts.ppSupportFuse) dec->ppSupport = PP_NOT_SUPPORTED;
2124
2125                                 /* check the pp config vs fuse status */
2126                                 if ((dec->ppConfig & 0xFC000000) && ((hwFuseSts.ppConfigFuse & 0xF0000000) >> 5)) {
2127                                         u32 deInterlace = ((dec->ppConfig & PP_DEINTERLACING) >> 25);
2128                                         u32 alphaBlend  = ((dec->ppConfig & PP_ALPHA_BLENDING) >> 24);
2129                                         u32 deInterlaceFuse = (((hwFuseSts.ppConfigFuse >> 5) & PP_DEINTERLACING) >> 25);
2130                                         u32 alphaBlendFuse  = (((hwFuseSts.ppConfigFuse >> 5) & PP_ALPHA_BLENDING) >> 24);
2131
2132                                         if (deInterlace && !deInterlaceFuse) dec->ppConfig &= 0xFD000000;
2133                                         if (alphaBlend && !alphaBlendFuse) dec->ppConfig &= 0xFE000000;
2134                                 }
2135                                 if (!hwFuseSts.sorensonSparkSupportFuse) dec->sorensonSparkSupport = SORENSON_SPARK_NOT_SUPPORTED;
2136                                 if (!hwFuseSts.refBufSupportFuse)   dec->refBufSupport = REF_BUF_NOT_SUPPORTED;
2137                                 if (!hwFuseSts.rvSupportFuse)       dec->rvSupport = RV_NOT_SUPPORTED;
2138                                 if (!hwFuseSts.avsSupportFuse)      dec->avsSupport = AVS_NOT_SUPPORTED;
2139                                 if (!hwFuseSts.mvcSupportFuse)      dec->mvcSupport = MVC_NOT_SUPPORTED;
2140                         }
2141                 }
2142
2143                 if (!cpu_is_rk3036()) {
2144                         configReg = pservice->enc_dev.hwregs[63];
2145                         enc->maxEncodedWidth = configReg & ((1 << 11) - 1);
2146                         enc->h264Enabled = (configReg >> 27) & 1;
2147                         enc->mpeg4Enabled = (configReg >> 26) & 1;
2148                         enc->jpegEnabled = (configReg >> 25) & 1;
2149                         enc->vsEnabled = (configReg >> 24) & 1;
2150                         enc->rgbEnabled = (configReg >> 28) & 1;
2151                         /*enc->busType = (configReg >> 20) & 15;
2152                         enc->synthesisLanguage = (configReg >> 16) & 15;
2153                         enc->busWidth = (configReg >> 12) & 15;*/
2154                         enc->reg_size = pservice->reg_size;
2155                         enc->reserv[0] = enc->reserv[1] = 0;
2156                 }
2157
2158                 pservice->auto_freq = soc_is_rk2928g() || soc_is_rk2928l() || soc_is_rk2926() || soc_is_rk3288();
2159                 if (pservice->auto_freq) {
2160                         pr_info("vpu_service set to auto frequency mode\n");
2161                         atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2162                 }
2163
2164                 pservice->bug_dec_addr = cpu_is_rk30xx();
2165         } else {
2166                 if (cpu_is_rk3036()  || cpu_is_rk312x())
2167                         dec->maxDecPicWidth = 1920;
2168                 else
2169                         dec->maxDecPicWidth = 4096;
2170                 /* disable frequency switch in hevc.*/
2171                 pservice->auto_freq = false;
2172         }
2173 }
2174
2175 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2176 {
2177         struct vpu_service_info *pservice = (struct vpu_service_info*)dev_id;
2178         vpu_device *dev = &pservice->dec_dev;
2179         u32 raw_status;
2180         u32 irq_status;
2181
2182         vcodec_enter_mode_nolock(pservice->dev_id, &pservice->reserved_mode);
2183
2184         irq_status = raw_status = readl(dev->hwregs + DEC_INTERRUPT_REGISTER);
2185
2186         pr_debug("dec_irq\n");
2187
2188         if (irq_status & DEC_INTERRUPT_BIT) {
2189                 pr_debug("dec_isr dec %x\n", irq_status);
2190                 if ((irq_status & 0x40001) == 0x40001)
2191                 {
2192                         do {
2193                                 irq_status = readl(dev->hwregs + DEC_INTERRUPT_REGISTER);
2194                         } while ((irq_status & 0x40001) == 0x40001);
2195                 }
2196
2197                 /* clear dec IRQ */
2198                 if (pservice->hw_info->hw_id != HEVC_ID)
2199                         writel(irq_status & (~DEC_INTERRUPT_BIT|DEC_BUFFER_EMPTY_BIT), dev->hwregs + DEC_INTERRUPT_REGISTER);
2200                 else
2201                         writel(0, dev->hwregs + DEC_INTERRUPT_REGISTER);
2202                 atomic_add(1, &dev->irq_count_codec);
2203         }
2204
2205         if (pservice->hw_info->hw_id != HEVC_ID) {
2206                 irq_status = readl(dev->hwregs + PP_INTERRUPT_REGISTER);
2207                 if (irq_status & PP_INTERRUPT_BIT) {
2208                         pr_debug("vdpu_isr pp  %x\n", irq_status);
2209                         /* clear pp IRQ */
2210                         writel(irq_status & (~DEC_INTERRUPT_BIT), dev->hwregs + PP_INTERRUPT_REGISTER);
2211                         atomic_add(1, &dev->irq_count_pp);
2212                 }
2213         }
2214
2215         pservice->irq_status = raw_status;
2216
2217         vcodec_exit_mode_nolock(pservice->dev_id, pservice->reserved_mode);
2218
2219         return IRQ_WAKE_THREAD;
2220 }
2221
2222 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2223 {
2224         struct vpu_service_info *pservice = (struct vpu_service_info*)dev_id;
2225         vpu_device *dev = &pservice->dec_dev;
2226         mutex_lock(&pservice->lock);
2227         if (atomic_read(&dev->irq_count_codec)) {
2228 #if VPU_SERVICE_SHOW_TIME
2229                 do_gettimeofday(&dec_end);
2230                 pr_info("dec task: %ld ms\n",
2231                         (dec_end.tv_sec  - dec_start.tv_sec)  * 1000 +
2232                         (dec_end.tv_usec - dec_start.tv_usec) / 1000);
2233 #endif
2234                 atomic_sub(1, &dev->irq_count_codec);
2235                 if (NULL == pservice->reg_codec) {
2236                         pr_err("error: dec isr with no task waiting\n");
2237                 } else {
2238                         reg_from_run_to_done(pservice, pservice->reg_codec);
2239                 }
2240         }
2241
2242         if (atomic_read(&dev->irq_count_pp)) {
2243
2244 #if VPU_SERVICE_SHOW_TIME
2245                 do_gettimeofday(&pp_end);
2246                 printk("pp  task: %ld ms\n",
2247                         (pp_end.tv_sec  - pp_start.tv_sec)  * 1000 +
2248                         (pp_end.tv_usec - pp_start.tv_usec) / 1000);
2249 #endif
2250
2251                 atomic_sub(1, &dev->irq_count_pp);
2252                 if (NULL == pservice->reg_pproc) {
2253                         pr_err("error: pp isr with no task waiting\n");
2254                 } else {
2255                         reg_from_run_to_done(pservice, pservice->reg_pproc);
2256                 }
2257         }
2258         try_set_reg(pservice);
2259         mutex_unlock(&pservice->lock);
2260         return IRQ_HANDLED;
2261 }
2262
2263 static irqreturn_t vepu_irq(int irq, void *dev_id)
2264 {
2265         struct vpu_service_info *pservice = (struct vpu_service_info*)dev_id;
2266         vpu_device *dev = &pservice->enc_dev;
2267         u32 irq_status;
2268
2269         vcodec_enter_mode_nolock(pservice->dev_id,  &pservice->reserved_mode);
2270         irq_status= readl(dev->hwregs + ENC_INTERRUPT_REGISTER);
2271
2272         pr_debug("vepu_irq irq status %x\n", irq_status);
2273
2274 #if VPU_SERVICE_SHOW_TIME
2275         do_gettimeofday(&enc_end);
2276         pr_info("enc task: %ld ms\n",
2277                 (enc_end.tv_sec  - enc_start.tv_sec)  * 1000 +
2278                 (enc_end.tv_usec - enc_start.tv_usec) / 1000);
2279 #endif
2280         if (likely(irq_status & ENC_INTERRUPT_BIT)) {
2281                 /* clear enc IRQ */
2282                 writel(irq_status & (~ENC_INTERRUPT_BIT), dev->hwregs + ENC_INTERRUPT_REGISTER);
2283                 atomic_add(1, &dev->irq_count_codec);
2284         }
2285
2286         pservice->irq_status = irq_status;
2287
2288         vcodec_exit_mode_nolock(pservice->dev_id, pservice->reserved_mode);
2289
2290         return IRQ_WAKE_THREAD;
2291 }
2292
2293 static irqreturn_t vepu_isr(int irq, void *dev_id)
2294 {
2295         struct vpu_service_info *pservice = (struct vpu_service_info*)dev_id;
2296         vpu_device *dev = &pservice->enc_dev;
2297
2298         mutex_lock(&pservice->lock);
2299         if (atomic_read(&dev->irq_count_codec)) {
2300                 atomic_sub(1, &dev->irq_count_codec);
2301                 if (NULL == pservice->reg_codec) {
2302                         pr_err("error: enc isr with no task waiting\n");
2303                 } else {
2304                         reg_from_run_to_done(pservice, pservice->reg_codec);
2305                 }
2306         }
2307         try_set_reg(pservice);
2308         mutex_unlock(&pservice->lock);
2309         return IRQ_HANDLED;
2310 }
2311
2312 static int __init vcodec_service_init(void)
2313 {
2314         int ret;
2315
2316         if ((ret = platform_driver_register(&vcodec_driver)) != 0) {
2317                 pr_err("Platform device register failed (%d).\n", ret);
2318                 return ret;
2319         }
2320
2321 #ifdef CONFIG_DEBUG_FS
2322         vcodec_debugfs_init();
2323 #endif
2324
2325         return ret;
2326 }
2327
2328 static void __exit vcodec_service_exit(void)
2329 {
2330 #ifdef CONFIG_DEBUG_FS
2331         vcodec_debugfs_exit();
2332 #endif
2333
2334         platform_driver_unregister(&vcodec_driver);
2335 }
2336
2337 module_init(vcodec_service_init);
2338 module_exit(vcodec_service_exit);
2339
2340 #ifdef CONFIG_DEBUG_FS
2341 #include <linux/seq_file.h>
2342
2343 static int vcodec_debugfs_init()
2344 {
2345         parent = debugfs_create_dir("vcodec", NULL);
2346         if (!parent)
2347                 return -1;
2348
2349         return 0;
2350 }
2351
2352 static void vcodec_debugfs_exit()
2353 {
2354         debugfs_remove(parent);
2355 }
2356
2357 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent)
2358 {
2359         return debugfs_create_dir(dirname, parent);
2360 }
2361
2362 static int debug_vcodec_show(struct seq_file *s, void *unused)
2363 {
2364         struct vpu_service_info *pservice = s->private;
2365         unsigned int i, n;
2366         vpu_reg *reg, *reg_tmp;
2367         vpu_session *session, *session_tmp;
2368
2369         mutex_lock(&pservice->lock);
2370         vpu_service_power_on(pservice);
2371         if (pservice->hw_info->hw_id != HEVC_ID) {
2372                 seq_printf(s, "\nENC Registers:\n");
2373                 n = pservice->enc_dev.iosize >> 2;
2374                 for (i = 0; i < n; i++) {
2375                         seq_printf(s, "\tswreg%d = %08X\n", i, readl(pservice->enc_dev.hwregs + i));
2376                 }
2377         }
2378         seq_printf(s, "\nDEC Registers:\n");
2379         n = pservice->dec_dev.iosize >> 2;
2380         for (i = 0; i < n; i++)
2381                 seq_printf(s, "\tswreg%d = %08X\n", i, readl(pservice->dec_dev.hwregs + i));
2382
2383         seq_printf(s, "\nvpu service status:\n");
2384         list_for_each_entry_safe(session, session_tmp, &pservice->session, list_session) {
2385                 seq_printf(s, "session pid %d type %d:\n", session->pid, session->type);
2386                 /*seq_printf(s, "waiting reg set %d\n");*/
2387                 list_for_each_entry_safe(reg, reg_tmp, &session->waiting, session_link) {
2388                         seq_printf(s, "waiting register set\n");
2389                 }
2390                 list_for_each_entry_safe(reg, reg_tmp, &session->running, session_link) {
2391                         seq_printf(s, "running register set\n");
2392                 }
2393                 list_for_each_entry_safe(reg, reg_tmp, &session->done, session_link) {
2394                         seq_printf(s, "done    register set\n");
2395                 }
2396         }
2397         mutex_unlock(&pservice->lock);
2398
2399         return 0;
2400 }
2401
2402 static int debug_vcodec_open(struct inode *inode, struct file *file)
2403 {
2404         return single_open(file, debug_vcodec_show, inode->i_private);
2405 }
2406
2407 #endif
2408
2409 #if HEVC_TEST_ENABLE & defined(CONFIG_ION_ROCKCHIP)
2410 #include "hevc_test_inc/pps_00.h"
2411 #include "hevc_test_inc/register_00.h"
2412 #include "hevc_test_inc/rps_00.h"
2413 #include "hevc_test_inc/scaling_list_00.h"
2414 #include "hevc_test_inc/stream_00.h"
2415
2416 #include "hevc_test_inc/pps_01.h"
2417 #include "hevc_test_inc/register_01.h"
2418 #include "hevc_test_inc/rps_01.h"
2419 #include "hevc_test_inc/scaling_list_01.h"
2420 #include "hevc_test_inc/stream_01.h"
2421
2422 #include "hevc_test_inc/cabac.h"
2423
2424 extern struct ion_client *rockchip_ion_client_create(const char * name);
2425
2426 static struct ion_client *ion_client = NULL;
2427 u8* get_align_ptr(u8* tbl, int len, u32 *phy)
2428 {
2429         int size = (len+15) & (~15);
2430         struct ion_handle *handle;
2431         u8 *ptr;// = (u8*)kzalloc(size, GFP_KERNEL);
2432
2433         if (ion_client == NULL)
2434                 ion_client = rockchip_ion_client_create("vcodec");
2435
2436         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2437
2438         ptr = ion_map_kernel(ion_client, handle);
2439
2440         ion_phys(ion_client, handle, phy, &size);
2441
2442         memcpy(ptr, tbl, len);
2443
2444         return ptr;
2445 }
2446
2447 u8* get_align_ptr_no_copy(int len, u32 *phy)
2448 {
2449         int size = (len+15) & (~15);
2450         struct ion_handle *handle;
2451         u8 *ptr;
2452
2453         if (ion_client == NULL)
2454                 ion_client = rockchip_ion_client_create("vcodec");
2455
2456         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2457
2458         ptr = ion_map_kernel(ion_client, handle);
2459
2460         ion_phys(ion_client, handle, phy, &size);
2461
2462         return ptr;
2463 }
2464
2465 #define TEST_CNT    2
2466 static int hevc_test_case0(vpu_service_info *pservice)
2467 {
2468         vpu_session session;
2469         vpu_reg *reg;
2470         unsigned long size = 272;//sizeof(register_00); // registers array length
2471         int testidx = 0;
2472         int ret = 0;
2473
2474         u8 *pps_tbl[TEST_CNT];
2475         u8 *register_tbl[TEST_CNT];
2476         u8 *rps_tbl[TEST_CNT];
2477         u8 *scaling_list_tbl[TEST_CNT];
2478         u8 *stream_tbl[TEST_CNT];
2479
2480         int stream_size[2];
2481         int pps_size[2];
2482         int rps_size[2];
2483         int scl_size[2];
2484         int cabac_size[2];
2485
2486         u32 phy_pps;
2487         u32 phy_rps;
2488         u32 phy_scl;
2489         u32 phy_str;
2490         u32 phy_yuv;
2491         u32 phy_ref;
2492         u32 phy_cabac;
2493
2494         volatile u8 *stream_buf;
2495         volatile u8 *pps_buf;
2496         volatile u8 *rps_buf;
2497         volatile u8 *scl_buf;
2498         volatile u8 *yuv_buf;
2499         volatile u8 *cabac_buf;
2500         volatile u8 *ref_buf;
2501
2502         u8 *pps;
2503         u8 *yuv[2];
2504         int i;
2505
2506         pps_tbl[0] = pps_00;
2507         pps_tbl[1] = pps_01;
2508
2509         register_tbl[0] = register_00;
2510         register_tbl[1] = register_01;
2511
2512         rps_tbl[0] = rps_00;
2513         rps_tbl[1] = rps_01;
2514
2515         scaling_list_tbl[0] = scaling_list_00;
2516         scaling_list_tbl[1] = scaling_list_01;
2517
2518         stream_tbl[0] = stream_00;
2519         stream_tbl[1] = stream_01;
2520
2521         stream_size[0] = sizeof(stream_00);
2522         stream_size[1] = sizeof(stream_01);
2523
2524         pps_size[0] = sizeof(pps_00);
2525         pps_size[1] = sizeof(pps_01);
2526
2527         rps_size[0] = sizeof(rps_00);
2528         rps_size[1] = sizeof(rps_01);
2529
2530         scl_size[0] = sizeof(scaling_list_00);
2531         scl_size[1] = sizeof(scaling_list_01);
2532
2533         cabac_size[0] = sizeof(Cabac_table);
2534         cabac_size[1] = sizeof(Cabac_table);
2535
2536         /* create session */
2537         session.pid = current->pid;
2538         session.type = VPU_DEC;
2539         INIT_LIST_HEAD(&session.waiting);
2540         INIT_LIST_HEAD(&session.running);
2541         INIT_LIST_HEAD(&session.done);
2542         INIT_LIST_HEAD(&session.list_session);
2543         init_waitqueue_head(&session.wait);
2544         atomic_set(&session.task_running, 0);
2545         list_add_tail(&session.list_session, &pservice->session);
2546
2547         yuv[0] = get_align_ptr_no_copy(256*256*2, &phy_yuv);
2548         yuv[1] = get_align_ptr_no_copy(256*256*2, &phy_ref);
2549
2550         while (testidx < TEST_CNT) {
2551                 /* create registers */
2552                 reg = kmalloc(sizeof(vpu_reg)+pservice->reg_size, GFP_KERNEL);
2553                 if (NULL == reg) {
2554                         pr_err("error: kmalloc fail in reg_init\n");
2555                         return -1;
2556                 }
2557
2558                 if (size > pservice->reg_size) {
2559                         printk("warning: vpu reg size %lu is larger than hw reg size %lu\n", size, pservice->reg_size);
2560                         size = pservice->reg_size;
2561                 }
2562                 reg->session = &session;
2563                 reg->type = session.type;
2564                 reg->size = size;
2565                 reg->freq = VPU_FREQ_DEFAULT;
2566                 reg->reg = (unsigned long *)&reg[1];
2567                 INIT_LIST_HEAD(&reg->session_link);
2568                 INIT_LIST_HEAD(&reg->status_link);
2569
2570                 /* TODO: stuff registers */
2571                 memcpy(&reg->reg[0], register_tbl[testidx], /*sizeof(register_00)*/ 176);
2572
2573                 stream_buf = get_align_ptr(stream_tbl[testidx], stream_size[testidx], &phy_str);
2574                 pps_buf = get_align_ptr(pps_tbl[0], pps_size[0], &phy_pps);
2575                 rps_buf = get_align_ptr(rps_tbl[testidx], rps_size[testidx], &phy_rps);
2576                 scl_buf = get_align_ptr(scaling_list_tbl[testidx], scl_size[testidx], &phy_scl);
2577                 cabac_buf = get_align_ptr(Cabac_table, cabac_size[testidx], &phy_cabac);
2578
2579                 pps = pps_buf;
2580
2581                 /* TODO: replace reigster address */
2582                 for (i=0; i<64; i++) {
2583                         u32 scaling_offset;
2584                         u32 tmp;
2585
2586                         scaling_offset = (u32)pps[i*80+74];
2587                         scaling_offset += (u32)pps[i*80+75] << 8;
2588                         scaling_offset += (u32)pps[i*80+76] << 16;
2589                         scaling_offset += (u32)pps[i*80+77] << 24;
2590
2591                         tmp = phy_scl + scaling_offset;
2592
2593                         pps[i*80+74] = tmp & 0xff;
2594                         pps[i*80+75] = (tmp >> 8) & 0xff;
2595                         pps[i*80+76] = (tmp >> 16) & 0xff;
2596                         pps[i*80+77] = (tmp >> 24) & 0xff;
2597                 }
2598
2599                 printk("%s %d, phy stream %08x, phy pps %08x, phy rps %08x\n",
2600                         __func__, __LINE__, phy_str, phy_pps, phy_rps);
2601
2602                 reg->reg[1] = 0x21;
2603                 reg->reg[4] = phy_str;
2604                 reg->reg[5] = ((stream_size[testidx]+15)&(~15))+64;
2605                 reg->reg[6] = phy_cabac;
2606                 reg->reg[7] = testidx?phy_ref:phy_yuv;
2607                 reg->reg[42] = phy_pps;
2608                 reg->reg[43] = phy_rps;
2609                 for (i = 10; i <= 24; i++)
2610                         reg->reg[i] = phy_yuv;
2611
2612                 mutex_lock(&pservice->lock);
2613                 list_add_tail(&reg->status_link, &pservice->waiting);
2614                 list_add_tail(&reg->session_link, &session.waiting);
2615                 mutex_unlock(&pservice->lock);
2616
2617                 printk("%s %d %p\n", __func__, __LINE__, pservice);
2618
2619                 /* stuff hardware */
2620                 try_set_reg(pservice);
2621
2622                 /* wait for result */
2623                 ret = wait_event_timeout(session.wait, !list_empty(&session.done), VPU_TIMEOUT_DELAY);
2624                 if (!list_empty(&session.done)) {
2625                         if (ret < 0)
2626                                 pr_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session.pid, ret);
2627                         ret = 0;
2628                 } else {
2629                         if (unlikely(ret < 0)) {
2630                                 pr_err("error: pid %d wait task ret %d\n", session.pid, ret);
2631                         } else if (0 == ret) {
2632                                 pr_err("error: pid %d wait %d task done timeout\n", session.pid, atomic_read(&session.task_running));
2633                                 ret = -ETIMEDOUT;
2634                         }
2635                 }
2636                 if (ret < 0) {
2637                         int task_running = atomic_read(&session.task_running);
2638                         int n;
2639                         mutex_lock(&pservice->lock);
2640                         vpu_service_dump(pservice);
2641                         if (task_running) {
2642                                 atomic_set(&session.task_running, 0);
2643                                 atomic_sub(task_running, &pservice->total_running);
2644                                 printk("%d task is running but not return, reset hardware...", task_running);
2645                                 vpu_reset(pservice);
2646                                 printk("done\n");
2647                         }
2648                         vpu_service_session_clear(pservice, &session);
2649                         mutex_unlock(&pservice->lock);
2650
2651                         printk("\nDEC Registers:\n");
2652                         n = pservice->dec_dev.iosize >> 2;
2653                         for (i=0; i<n; i++)
2654                                 printk("\tswreg%d = %08X\n", i, readl(pservice->dec_dev.hwregs + i));
2655
2656                         pr_err("test index %d failed\n", testidx);
2657                         break;
2658                 } else {
2659                         pr_info("test index %d success\n", testidx);
2660
2661                         vpu_reg *reg = list_entry(session.done.next, vpu_reg, session_link);
2662
2663                         for (i=0; i<68; i++) {
2664                                 if (i % 4 == 0)
2665                                         printk("%02d: ", i);
2666                                 printk("%08x ", reg->reg[i]);
2667                                 if ((i+1) % 4 == 0)
2668                                         printk("\n");
2669                         }
2670
2671                         testidx++;
2672                 }
2673
2674                 reg_deinit(pservice, reg);
2675         }
2676
2677         return 0;
2678 }
2679
2680 #endif
2681