drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/clk.h>
  20 #include <linux/compat.h>
  21 #include <linux/delay.h>
  22 #include <linux/init.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/module.h>
  25 #include <linux/fs.h>
  26 #include <linux/mm.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/reset.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31 #include <linux/wakelock.h>
  32 #include <linux/cdev.h>
  33 #include <linux/of.h>
  34 #include <linux/of_platform.h>
  35 #include <linux/of_irq.h>
  36 #include <linux/regmap.h>
  37 #include <linux/mfd/syscon.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/debugfs.h>
  40
  41 #include <linux/rockchip/cpu.h>
  42 #include <linux/rockchip/cru.h>
  43 #include <linux/rockchip/pmu.h>
  44 #include <linux/rockchip/grf.h>
  45
  46 #if defined(CONFIG_ION_ROCKCHIP)
  47 #include <linux/rockchip_ion.h>
  48 #endif
  49
  50 #include <linux/rockchip-iovmm.h>
  51 #include <linux/dma-buf.h>
  52
  53 #include "vcodec_hw_info.h"
  54 #include "vcodec_hw_vpu.h"
  55 #include "vcodec_hw_rkv.h"
  56 #include "vcodec_hw_vpu2.h"
  57
  58 #include "vcodec_service.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 static int debug;
  90 module_param(debug, int, S_IRUGO | S_IWUSR);
  91 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  92
  93 #define VCODEC_CLOCK_ENABLE     1
  94
  95 /*
  96  * hardware information organization
  97  *
  98  * In order to support multiple hardware with different version the hardware
  99  * information is organized as follow:
 100  *
 101  * 1. First, index hardware by register size / position.
 102  *    These information is fix for each hardware and do not relate to runtime
 103  *    work flow. It only related to resource allocation.
 104  *    Descriptor: struct vpu_hw_info
 105  *
 106  * 2. Then, index hardware by runtime configuration
 107  *    These information is related to runtime setting behave including enable
 108  *    register, irq register and other key control flag
 109  *    Descriptor: struct vpu_task_info
 110  *
 111  * 3. Final, on iommu case the fd translation is required
 112  *    Descriptor: struct vpu_trans_info
 113  */
 114
 115 enum VPU_FREQ {
 116         VPU_FREQ_200M,
 117         VPU_FREQ_266M,
 118         VPU_FREQ_300M,
 119         VPU_FREQ_400M,
 120         VPU_FREQ_500M,
 121         VPU_FREQ_600M,
 122         VPU_FREQ_DEFAULT,
 123         VPU_FREQ_BUT,
 124 };
 125
 126 struct extra_info_elem {
 127         u32 index;
 128         u32 offset;
 129 };
 130
 131 #define EXTRA_INFO_MAGIC        0x4C4A46
 132
 133 struct extra_info_for_iommu {
 134         u32 magic;
 135         u32 cnt;
 136         struct extra_info_elem elem[20];
 137 };
 138
 139 #define MHZ                                     (1000*1000)
 140 #define SIZE_REG(reg)                           ((reg)*4)
 141
 142 static struct vcodec_info vcodec_info_set[] = {
 143         [0] = {
 144                 .hw_id          = VPU_ID_8270,
 145                 .hw_info        = &hw_vpu_8270,
 146                 .task_info      = task_vpu,
 147                 .trans_info     = trans_vpu,
 148         },
 149         [1] = {
 150                 .hw_id          = VPU_ID_4831,
 151                 .hw_info        = &hw_vpu_4831,
 152                 .task_info      = task_vpu,
 153                 .trans_info     = trans_vpu,
 154         },
 155         [2] = {
 156                 .hw_id          = VPU_DEC_ID_9190,
 157                 .hw_info        = &hw_vpu_9190,
 158                 .task_info      = task_vpu,
 159                 .trans_info     = trans_vpu,
 160         },
 161         [3] = {
 162                 .hw_id          = HEVC_ID,
 163                 .hw_info        = &hw_rkhevc,
 164                 .task_info      = task_rkv,
 165                 .trans_info     = trans_rkv,
 166         },
 167         [4] = {
 168                 .hw_id          = RKV_DEC_ID,
 169                 .hw_info        = &hw_rkvdec,
 170                 .task_info      = task_rkv,
 171                 .trans_info     = trans_rkv,
 172         },
 173         [5] = {
 174                 .hw_id          = VPU2_ID,
 175                 .hw_info        = &hw_vpu2,
 176                 .task_info      = task_vpu2,
 177                 .trans_info     = trans_vpu2,
 178         },
 179 };
 180
 181 #define DEBUG
 182 #ifdef DEBUG
 183 #define vpu_debug_func(type, fmt, args...)                      \
 184         do {                                                    \
 185                 if (unlikely(debug & type)) {                   \
 186                         pr_info("%s:%d: " fmt,                  \
 187                                  __func__, __LINE__, ##args);   \
 188                 }                                               \
 189         } while (0)
 190 #define vpu_debug(type, fmt, args...)                           \
 191         do {                                                    \
 192                 if (unlikely(debug & type)) {                   \
 193                         pr_info(fmt, ##args);                   \
 194                 }                                               \
 195         } while (0)
 196 #else
 197 #define vpu_debug_func(level, fmt, args...)
 198 #define vpu_debug(level, fmt, args...)
 199 #endif
 200
 201 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 202 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 203
 204 #define vpu_err(fmt, args...)                           \
 205                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 206
 207 enum VPU_DEC_FMT {
 208         VPU_DEC_FMT_H264,
 209         VPU_DEC_FMT_MPEG4,
 210         VPU_DEC_FMT_H263,
 211         VPU_DEC_FMT_JPEG,
 212         VPU_DEC_FMT_VC1,
 213         VPU_DEC_FMT_MPEG2,
 214         VPU_DEC_FMT_MPEG1,
 215         VPU_DEC_FMT_VP6,
 216         VPU_DEC_FMT_RESERV0,
 217         VPU_DEC_FMT_VP7,
 218         VPU_DEC_FMT_VP8,
 219         VPU_DEC_FMT_AVS,
 220         VPU_DEC_FMT_RES
 221 };
 222
 223 /**
 224  * struct for process session which connect to vpu
 225  *
 226  * @author ChenHengming (2011-5-3)
 227  */
 228 struct vpu_session {
 229         enum VPU_CLIENT_TYPE type;
 230         /* a linked list of data so we can access them for debugging */
 231         struct list_head list_session;
 232         /* a linked list of register data waiting for process */
 233         struct list_head waiting;
 234         /* a linked list of register data in processing */
 235         struct list_head running;
 236         /* a linked list of register data processed */
 237         struct list_head done;
 238         wait_queue_head_t wait;
 239         pid_t pid;
 240         atomic_t task_running;
 241 };
 242
 243 /**
 244  * struct for process register set
 245  *
 246  * @author ChenHengming (2011-5-4)
 247  */
 248 struct vpu_reg {
 249         enum VPU_CLIENT_TYPE type;
 250         enum VPU_FREQ freq;
 251         struct vpu_session *session;
 252         struct vpu_subdev_data *data;
 253         struct vpu_task_info *task;
 254         const struct vpu_trans_info *trans;
 255
 256         /* link to vpu service session */
 257         struct list_head session_link;
 258         /* link to register set list */
 259         struct list_head status_link;
 260
 261         unsigned long size;
 262         struct list_head mem_region_list;
 263         u32 dec_base;
 264         u32 *reg;
 265 };
 266
 267 struct vpu_device {
 268         atomic_t irq_count_codec;
 269         atomic_t irq_count_pp;
 270         unsigned int iosize;
 271         u32 *regs;
 272 };
 273
 274 enum vcodec_device_id {
 275         VCODEC_DEVICE_ID_VPU,
 276         VCODEC_DEVICE_ID_HEVC,
 277         VCODEC_DEVICE_ID_COMBO,
 278         VCODEC_DEVICE_ID_RKVDEC,
 279         VCODEC_DEVICE_ID_BUTT
 280 };
 281
 282 enum VCODEC_RUNNING_MODE {
 283         VCODEC_RUNNING_MODE_NONE = -1,
 284         VCODEC_RUNNING_MODE_VPU,
 285         VCODEC_RUNNING_MODE_HEVC,
 286         VCODEC_RUNNING_MODE_RKVDEC
 287 };
 288
 289 struct vcodec_mem_region {
 290         struct list_head srv_lnk;
 291         struct list_head reg_lnk;
 292         struct list_head session_lnk;
 293         unsigned long iova;     /* virtual address for iommu */
 294         unsigned long len;
 295         u32 reg_idx;
 296         struct ion_handle *hdl;
 297 };
 298
 299 enum vpu_ctx_state {
 300         MMU_ACTIVATED   = BIT(0)
 301 };
 302
 303 struct vpu_subdev_data {
 304         struct cdev cdev;
 305         dev_t dev_t;
 306         struct class *cls;
 307         struct device *child_dev;
 308
 309         int irq_enc;
 310         int irq_dec;
 311         struct vpu_service_info *pservice;
 312
 313         u32 *regs;
 314         enum VCODEC_RUNNING_MODE mode;
 315         struct list_head lnk_service;
 316
 317         struct device *dev;
 318
 319         struct vpu_device enc_dev;
 320         struct vpu_device dec_dev;
 321
 322         enum VPU_HW_ID hw_id;
 323         struct vpu_hw_info *hw_info;
 324         struct vpu_task_info *task_info;
 325         const struct vpu_trans_info *trans_info;
 326
 327         u32 reg_size;
 328         unsigned long state;
 329
 330 #ifdef CONFIG_DEBUG_FS
 331         struct dentry *debugfs_dir;
 332         struct dentry *debugfs_file_regs;
 333 #endif
 334
 335         struct device *mmu_dev;
 336 };
 337
 338 struct vpu_service_info {
 339         struct wake_lock wake_lock;
 340         struct delayed_work power_off_work;
 341         /* vpu service structure global lock */
 342         struct mutex lock;
 343         /* link to link_reg in struct vpu_reg */
 344         struct list_head waiting;
 345         /* link to link_reg in struct vpu_reg */
 346         struct list_head running;
 347         /* link to link_reg in struct vpu_reg */
 348         struct list_head done;
 349         /* link to list_session in struct vpu_session */
 350         struct list_head session;
 351         atomic_t total_running;
 352         atomic_t enabled;
 353         atomic_t power_on_cnt;
 354         atomic_t power_off_cnt;
 355         struct vpu_reg *reg_codec;
 356         struct vpu_reg *reg_pproc;
 357         struct vpu_reg *reg_resev;
 358         struct vpu_dec_config dec_config;
 359         struct vpu_enc_config enc_config;
 360
 361         bool auto_freq;
 362         bool bug_dec_addr;
 363         atomic_t freq_status;
 364
 365         struct clk *aclk_vcodec;
 366         struct clk *hclk_vcodec;
 367         struct clk *clk_core;
 368         struct clk *clk_cabac;
 369         struct clk *pd_video;
 370
 371 #ifdef CONFIG_RESET_CONTROLLER
 372         struct reset_control *rst_a;
 373         struct reset_control *rst_h;
 374         struct reset_control *rst_v;
 375 #endif
 376         struct device *dev;
 377
 378         u32 irq_status;
 379         atomic_t reset_request;
 380         struct ion_client *ion_client;
 381         struct list_head mem_region_list;
 382
 383         enum vcodec_device_id dev_id;
 384
 385         enum VCODEC_RUNNING_MODE curr_mode;
 386         u32 prev_mode;
 387
 388         struct delayed_work simulate_work;
 389
 390         u32 mode_bit;
 391         u32 mode_ctrl;
 392         u32 *reg_base;
 393         u32 ioaddr;
 394         struct regmap *grf;
 395         u32 *grf_base;
 396
 397         char *name;
 398
 399         u32 subcnt;
 400         struct list_head subdev_list;
 401 };
 402
 403 struct vpu_request {
 404         u32 *req;
 405         u32 size;
 406 };
 407
 408 #ifdef CONFIG_COMPAT
 409 struct compat_vpu_request {
 410         compat_uptr_t req;
 411         u32 size;
 412 };
 413 #endif
 414
 415 /* debugfs root directory for all device (vpu, hevc).*/
 416 static struct dentry *parent;
 417
 418 #ifdef CONFIG_DEBUG_FS
 419 static int vcodec_debugfs_init(void);
 420 static void vcodec_debugfs_exit(void);
 421 static struct dentry *vcodec_debugfs_create_device_dir(
 422                 char *dirname, struct dentry *parent);
 423 static int debug_vcodec_open(struct inode *inode, struct file *file);
 424
 425 static const struct file_operations debug_vcodec_fops = {
 426         .open = debug_vcodec_open,
 427         .read = seq_read,
 428         .llseek = seq_lseek,
 429         .release = single_release,
 430 };
 431 #endif
 432
 433 #define VDPU_SOFT_RESET_REG     101
 434 #define VDPU_CLEAN_CACHE_REG    516
 435 #define VEPU_CLEAN_CACHE_REG    772
 436 #define HEVC_CLEAN_CACHE_REG    260
 437
 438 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 439
 440 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 441 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 442 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 443 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 444
 445 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 446 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 447
 448 static void time_record(struct vpu_task_info *task, int is_end)
 449 {
 450         if (unlikely(debug & DEBUG_TIMING) && task)
 451                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 452 }
 453
 454 static void time_diff(struct vpu_task_info *task)
 455 {
 456         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 457                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 458                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 459 }
 460
 461 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 462 {
 463         int bits;
 464         u32 raw = 0;
 465         struct vpu_service_info *pservice = data->pservice;
 466         struct vpu_subdev_data *subdata, *n;
 467
 468         if (pservice->subcnt < 2) {
 469                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 470                         set_bit(MMU_ACTIVATED, &data->state);
 471                         if (atomic_read(&pservice->enabled))
 472                                 rockchip_iovmm_activate(data->dev);
 473                         else
 474                                 BUG_ON(!atomic_read(&pservice->enabled));
 475                 }
 476                 return;
 477         }
 478
 479         if (pservice->curr_mode == data->mode)
 480                 return;
 481
 482         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 483         list_for_each_entry_safe(subdata, n,
 484                                  &pservice->subdev_list, lnk_service) {
 485                 if (data != subdata && subdata->mmu_dev &&
 486                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 487                         clear_bit(MMU_ACTIVATED, &subdata->state);
 488                         rockchip_iovmm_deactivate(subdata->dev);
 489                 }
 490         }
 491         bits = 1 << pservice->mode_bit;
 492 #ifdef CONFIG_MFD_SYSCON
 493         if (pservice->grf) {
 494                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 495
 496                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 497                         regmap_write(pservice->grf, pservice->mode_ctrl,
 498                                      raw | bits | (bits << 16));
 499                 else
 500                         regmap_write(pservice->grf, pservice->mode_ctrl,
 501                                      (raw & (~bits)) | (bits << 16));
 502         } else if (pservice->grf_base) {
 503                 u32 *grf_base = pservice->grf_base;
 504
 505                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 506                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 507                         writel_relaxed(raw | bits | (bits << 16),
 508                                        grf_base + pservice->mode_ctrl / 4);
 509                 else
 510                         writel_relaxed((raw & (~bits)) | (bits << 16),
 511                                        grf_base + pservice->mode_ctrl / 4);
 512         } else {
 513                 vpu_err("no grf resource define, switch decoder failed\n");
 514                 return;
 515         }
 516 #else
 517         if (pservice->grf_base) {
 518                 u32 *grf_base = pservice->grf_base;
 519
 520                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 521                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 522                         writel_relaxed(raw | bits | (bits << 16),
 523                                        grf_base + pservice->mode_ctrl / 4);
 524                 else
 525                         writel_relaxed((raw & (~bits)) | (bits << 16),
 526                                        grf_base + pservice->mode_ctrl / 4);
 527         } else {
 528                 vpu_err("no grf resource define, switch decoder failed\n");
 529                 return;
 530         }
 531 #endif
 532         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 533                 set_bit(MMU_ACTIVATED, &data->state);
 534                 if (atomic_read(&pservice->enabled))
 535                         rockchip_iovmm_activate(data->dev);
 536                 else
 537                         BUG_ON(!atomic_read(&pservice->enabled));
 538         }
 539
 540         pservice->prev_mode = pservice->curr_mode;
 541         pservice->curr_mode = data->mode;
 542 }
 543
 544 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 545 {
 546         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 547                 clear_bit(MMU_ACTIVATED, &data->state);
 548                 rockchip_iovmm_deactivate(data->dev);
 549                 data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 550         }
 551 }
 552
 553 static int vpu_get_clk(struct vpu_service_info *pservice)
 554 {
 555 #if VCODEC_CLOCK_ENABLE
 556         struct device *dev = pservice->dev;
 557
 558         switch (pservice->dev_id) {
 559         case VCODEC_DEVICE_ID_HEVC:
 560                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 561                 if (IS_ERR(pservice->pd_video)) {
 562                         dev_err(dev, "failed on clk_get pd_hevc\n");
 563                         return -1;
 564                 }
 565         case VCODEC_DEVICE_ID_COMBO:
 566         case VCODEC_DEVICE_ID_RKVDEC:
 567                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 568                 if (IS_ERR(pservice->clk_cabac)) {
 569                         dev_err(dev, "failed on clk_get clk_cabac\n");
 570                         pservice->clk_cabac = NULL;
 571                 }
 572                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 573                 if (IS_ERR(pservice->clk_core)) {
 574                         dev_err(dev, "failed on clk_get clk_core\n");
 575                         return -1;
 576                 }
 577         case VCODEC_DEVICE_ID_VPU:
 578                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 579                 if (IS_ERR(pservice->aclk_vcodec)) {
 580                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 581                         return -1;
 582                 }
 583
 584                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 585                 if (IS_ERR(pservice->hclk_vcodec)) {
 586                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 587                         return -1;
 588                 }
 589                 if (pservice->pd_video == NULL) {
 590                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 591                         if (IS_ERR(pservice->pd_video)) {
 592                                 pservice->pd_video = NULL;
 593                                 dev_info(dev, "do not have pd_video\n");
 594                         }
 595                 }
 596                 break;
 597         default:
 598                 break;
 599         }
 600
 601         return 0;
 602 #else
 603         return 0;
 604 #endif
 605 }
 606
 607 static void vpu_put_clk(struct vpu_service_info *pservice)
 608 {
 609 #if VCODEC_CLOCK_ENABLE
 610         if (pservice->pd_video)
 611                 devm_clk_put(pservice->dev, pservice->pd_video);
 612         if (pservice->aclk_vcodec)
 613                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 614         if (pservice->hclk_vcodec)
 615                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 616         if (pservice->clk_core)
 617                 devm_clk_put(pservice->dev, pservice->clk_core);
 618         if (pservice->clk_cabac)
 619                 devm_clk_put(pservice->dev, pservice->clk_cabac);
 620 #endif
 621 }
 622
 623 static void vpu_reset(struct vpu_subdev_data *data)
 624 {
 625         struct vpu_service_info *pservice = data->pservice;
 626         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 627
 628         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 629                 type = IDLE_REQ_HEVC;
 630
 631         pr_info("%s: resetting...", dev_name(pservice->dev));
 632
 633 #if defined(CONFIG_ARCH_RK29)
 634         clk_disable(aclk_ddr_vepu);
 635         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 636         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 637         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 638         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 639         mdelay(10);
 640         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 641         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 642         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 643         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 644         clk_enable(aclk_ddr_vepu);
 645 #elif defined(CONFIG_ARCH_RK30)
 646         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 647         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 648         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 649         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 650         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 651         mdelay(1);
 652         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 653         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 654         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 655         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 656         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 657 #else
 658 #endif
 659         WARN_ON(pservice->reg_codec != NULL);
 660         WARN_ON(pservice->reg_pproc != NULL);
 661         WARN_ON(pservice->reg_resev != NULL);
 662         pservice->reg_codec = NULL;
 663         pservice->reg_pproc = NULL;
 664         pservice->reg_resev = NULL;
 665
 666         pr_info("for 3288/3368...");
 667 #if 0 //def CONFIG_RESET_CONTROLLER
 668         if (pservice->rst_a && pservice->rst_h) {
 669                 if (rockchip_pmu_ops.set_idle_request)
 670                         rockchip_pmu_ops.set_idle_request(type, true);
 671                 pr_info("reset in\n");
 672                 if (pservice->rst_v)
 673                         reset_control_assert(pservice->rst_v);
 674                 reset_control_assert(pservice->rst_a);
 675                 reset_control_assert(pservice->rst_h);
 676                 udelay(5);
 677                 reset_control_deassert(pservice->rst_h);
 678                 reset_control_deassert(pservice->rst_a);
 679                 if (pservice->rst_v)
 680                         reset_control_deassert(pservice->rst_v);
 681                 if (rockchip_pmu_ops.set_idle_request)
 682                         rockchip_pmu_ops.set_idle_request(type, false);
 683         }
 684 #endif
 685
 686         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 687                 clear_bit(MMU_ACTIVATED, &data->state);
 688                 if (atomic_read(&pservice->enabled))
 689                         rockchip_iovmm_deactivate(data->dev);
 690                 else
 691                         BUG_ON(!atomic_read(&pservice->enabled));
 692         }
 693
 694         atomic_set(&pservice->reset_request, 0);
 695         pr_info("done\n");
 696 }
 697
 698 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 699 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 700                                       struct vpu_session *session)
 701 {
 702         struct vpu_reg *reg, *n;
 703
 704         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 705                 reg_deinit(data, reg);
 706         }
 707         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 708                 reg_deinit(data, reg);
 709         }
 710         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 711                 reg_deinit(data, reg);
 712         }
 713 }
 714
 715 static void vpu_service_dump(struct vpu_service_info *pservice)
 716 {
 717 }
 718
 719
 720 static void vpu_service_power_off(struct vpu_service_info *pservice)
 721 {
 722         int total_running;
 723         struct vpu_subdev_data *data = NULL, *n;
 724         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 725
 726         if (!ret)
 727                 return;
 728
 729         total_running = atomic_read(&pservice->total_running);
 730         if (total_running) {
 731                 pr_alert("alert: power off when %d task running!!\n",
 732                          total_running);
 733                 mdelay(50);
 734                 pr_alert("alert: delay 50 ms for running task\n");
 735                 vpu_service_dump(pservice);
 736         }
 737
 738         pr_info("%s: power off...", dev_name(pservice->dev));
 739
 740         udelay(5);
 741
 742         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 743                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 744                         clear_bit(MMU_ACTIVATED, &data->state);
 745                         rockchip_iovmm_deactivate(data->dev);
 746                 }
 747         }
 748         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 749
 750 #if VCODEC_CLOCK_ENABLE
 751                 if (pservice->pd_video)
 752                         clk_disable_unprepare(pservice->pd_video);
 753                 if (pservice->hclk_vcodec)
 754                         clk_disable_unprepare(pservice->hclk_vcodec);
 755                 if (pservice->aclk_vcodec)
 756                         clk_disable_unprepare(pservice->aclk_vcodec);
 757                 if (pservice->clk_core)
 758                         clk_disable_unprepare(pservice->clk_core);
 759                 if (pservice->clk_cabac)
 760                         clk_disable_unprepare(pservice->clk_cabac);
 761 #endif
 762
 763         atomic_add(1, &pservice->power_off_cnt);
 764         wake_unlock(&pservice->wake_lock);
 765         pr_info("done\n");
 766 }
 767
 768 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 769 {
 770         queue_delayed_work(system_wq, &pservice->power_off_work,
 771                            VPU_POWER_OFF_DELAY);
 772 }
 773
 774 static void vpu_power_off_work(struct work_struct *work_s)
 775 {
 776         struct delayed_work *dlwork = container_of(work_s,
 777                         struct delayed_work, work);
 778         struct vpu_service_info *pservice = container_of(dlwork,
 779                         struct vpu_service_info, power_off_work);
 780
 781         if (mutex_trylock(&pservice->lock)) {
 782                 vpu_service_power_off(pservice);
 783                 mutex_unlock(&pservice->lock);
 784         } else {
 785                 /* Come back later if the device is busy... */
 786                 vpu_queue_power_off_work(pservice);
 787         }
 788 }
 789
 790 static void vpu_service_power_on(struct vpu_service_info *pservice)
 791 {
 792         int ret;
 793         static ktime_t last;
 794         ktime_t now = ktime_get();
 795
 796         if (ktime_to_ns(ktime_sub(now, last)) > NSEC_PER_SEC) {
 797                 cancel_delayed_work_sync(&pservice->power_off_work);
 798                 vpu_queue_power_off_work(pservice);
 799                 last = now;
 800         }
 801         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 802         if (!ret)
 803                 return;
 804
 805         pr_info("%s: power on\n", dev_name(pservice->dev));
 806
 807 #define BIT_VCODEC_CLK_SEL      (1<<10)
 808         if (cpu_is_rk312x())
 809                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 810                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 811                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 812
 813 #if VCODEC_CLOCK_ENABLE
 814         if (pservice->aclk_vcodec)
 815                 clk_prepare_enable(pservice->aclk_vcodec);
 816         if (pservice->hclk_vcodec)
 817                 clk_prepare_enable(pservice->hclk_vcodec);
 818         if (pservice->clk_core)
 819                 clk_prepare_enable(pservice->clk_core);
 820         if (pservice->clk_cabac)
 821                 clk_prepare_enable(pservice->clk_cabac);
 822         if (pservice->pd_video)
 823                 clk_prepare_enable(pservice->pd_video);
 824 #endif
 825
 826         udelay(5);
 827         atomic_add(1, &pservice->power_on_cnt);
 828         wake_lock(&pservice->wake_lock);
 829 }
 830
 831 static inline bool reg_check_interlace(struct vpu_reg *reg)
 832 {
 833         u32 type = (reg->reg[3] & (1 << 23));
 834
 835         return (type > 0);
 836 }
 837
 838 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 839 {
 840         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 841
 842         return type;
 843 }
 844
 845 static inline int reg_probe_width(struct vpu_reg *reg)
 846 {
 847         int width_in_mb = reg->reg[4] >> 23;
 848
 849         return width_in_mb * 16;
 850 }
 851
 852 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 853 {
 854         int y_virstride = reg->reg[8];
 855
 856         return y_virstride;
 857 }
 858
 859 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 860                              struct vpu_reg *reg, int fd)
 861 {
 862         struct vpu_service_info *pservice = data->pservice;
 863         struct ion_handle *hdl;
 864         int ret = 0;
 865         struct vcodec_mem_region *mem_region;
 866
 867         hdl = ion_import_dma_buf(pservice->ion_client, fd);
 868         if (IS_ERR(hdl)) {
 869                 vpu_err("import dma-buf from fd %d failed\n", fd);
 870                 return PTR_ERR(hdl);
 871         }
 872         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 873
 874         if (mem_region == NULL) {
 875                 vpu_err("allocate memory for iommu memory region failed\n");
 876                 ion_free(pservice->ion_client, hdl);
 877                 return -1;
 878         }
 879
 880         mem_region->hdl = hdl;
 881         if (data->mmu_dev)
 882                 ret = ion_map_iommu(data->dev, pservice->ion_client,
 883                                     mem_region->hdl, &mem_region->iova,
 884                                     &mem_region->len);
 885         else
 886                 ret = ion_phys(pservice->ion_client,
 887                                mem_region->hdl,
 888                                (ion_phys_addr_t *)&mem_region->iova,
 889                                (size_t *)&mem_region->len);
 890
 891         if (ret < 0) {
 892                 vpu_err("fd %d ion map iommu failed\n", fd);
 893                 kfree(mem_region);
 894                 ion_free(pservice->ion_client, hdl);
 895                 return ret;
 896         }
 897         INIT_LIST_HEAD(&mem_region->reg_lnk);
 898         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 899         return mem_region->iova;
 900 }
 901
 902 /*
 903  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 904  * it by pps id in video stream data.
 905  *
 906  * So we need to translate the address in iommu case. The address data is also
 907  * 10bit fd + 22bit offset mode.
 908  * Because userspace decoder do not give the pps id in the register file sets
 909  * kernel driver need to translate each scaling list address in pps buffer which
 910  * means 256 pps for H.264, 64 pps for H.265.
 911  *
 912  * In order to optimize the performance kernel driver ask userspace decoder to
 913  * set all scaling list address in pps buffer to the same one which will be used
 914  * on current decoding task. Then kernel driver can only translate the first
 915  * address then copy it all pps buffer.
 916  */
 917 static void fill_scaling_list_addr_in_pps(
 918                 struct vpu_subdev_data *data,
 919                 struct vpu_reg *reg,
 920                 char *pps,
 921                 int pps_info_count,
 922                 int pps_info_size,
 923                 int scaling_list_addr_offset)
 924 {
 925         int base = scaling_list_addr_offset;
 926         int scaling_fd = 0;
 927         u32 scaling_offset;
 928
 929         scaling_offset  = (u32)pps[base + 0];
 930         scaling_offset += (u32)pps[base + 1] << 8;
 931         scaling_offset += (u32)pps[base + 2] << 16;
 932         scaling_offset += (u32)pps[base + 3] << 24;
 933
 934         scaling_fd = scaling_offset & 0x3ff;
 935         scaling_offset = scaling_offset >> 10;
 936
 937         if (scaling_fd > 0) {
 938                 int i = 0;
 939                 u32 tmp = vcodec_fd_to_iova(data, reg, scaling_fd);
 940                 tmp += scaling_offset;
 941
 942                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 943                         pps[base + 0] = (tmp >>  0) & 0xff;
 944                         pps[base + 1] = (tmp >>  8) & 0xff;
 945                         pps[base + 2] = (tmp >> 16) & 0xff;
 946                         pps[base + 3] = (tmp >> 24) & 0xff;
 947                 }
 948         }
 949 }
 950
 951 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data, const u8 *tbl,
 952                                 int size, struct vpu_reg *reg,
 953                                 struct extra_info_for_iommu *ext_inf)
 954 {
 955         struct vpu_service_info *pservice = data->pservice;
 956         struct vpu_task_info *task = reg->task;
 957         enum FORMAT_TYPE type;
 958         struct ion_handle *hdl;
 959         int ret = 0;
 960         struct vcodec_mem_region *mem_region;
 961         int i;
 962         int offset = 0;
 963
 964         if (tbl == NULL || size <= 0) {
 965                 dev_err(pservice->dev, "input arguments invalidate\n");
 966                 return -1;
 967         }
 968
 969         if (task->get_fmt)
 970                 type = task->get_fmt(reg->reg);
 971         else {
 972                 pr_err("invalid task with NULL get_fmt\n");
 973                 return -1;
 974         }
 975
 976         for (i = 0; i < size; i++) {
 977                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
 978
 979                 /* if userspace do not set the fd at this register, skip */
 980                 if (usr_fd == 0)
 981                         continue;
 982
 983                 /*
 984                  * special offset scale case
 985                  *
 986                  * This translation is for fd + offset translation.
 987                  * One register has 32bits. We need to transfer both buffer file
 988                  * handle and the start address offset so we packet file handle
 989                  * and offset together using below format.
 990                  *
 991                  *  0~9  bit for buffer file handle range 0 ~ 1023
 992                  * 10~31 bit for offset range 0 ~ 4M
 993                  *
 994                  * But on 4K case the offset can be larger the 4M
 995                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
 996                  * But MPEG4 will use the same register for colmv and it do not
 997                  * need scale.
 998                  *
 999                  * RKVdec do not have this issue.
1000                  */
1001                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1002                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1003                         offset = reg->reg[tbl[i]] >> 10 << 4;
1004                 else
1005                         offset = reg->reg[tbl[i]] >> 10;
1006
1007                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d\n",
1008                           tbl[i], usr_fd, offset);
1009
1010                 hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
1011                 if (IS_ERR(hdl)) {
1012                         dev_err(pservice->dev,
1013                                 "import dma-buf from fd %d failed, reg[%d]\n",
1014                                 usr_fd, tbl[i]);
1015                         return PTR_ERR(hdl);
1016                 }
1017
1018                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1019                         int pps_info_offset;
1020                         int pps_info_count;
1021                         int pps_info_size;
1022                         int scaling_list_addr_offset;
1023
1024                         switch (type) {
1025                         case FMT_H264D: {
1026                                 pps_info_offset = offset;
1027                                 pps_info_count = 256;
1028                                 pps_info_size = 32;
1029                                 scaling_list_addr_offset = 23;
1030                         } break;
1031                         case FMT_H265D: {
1032                                 pps_info_offset = 0;
1033                                 pps_info_count = 64;
1034                                 pps_info_size = 80;
1035                                 scaling_list_addr_offset = 74;
1036                         } break;
1037                         default: {
1038                                 pps_info_offset = 0;
1039                                 pps_info_count = 0;
1040                                 pps_info_size = 0;
1041                                 scaling_list_addr_offset = 0;
1042                         } break;
1043                         }
1044
1045                         vpu_debug(DEBUG_PPS_FILL,
1046                                   "scaling list filling parameter:\n");
1047                         vpu_debug(DEBUG_PPS_FILL,
1048                                   "pps_info_offset %d\n", pps_info_offset);
1049                         vpu_debug(DEBUG_PPS_FILL,
1050                                   "pps_info_count  %d\n", pps_info_count);
1051                         vpu_debug(DEBUG_PPS_FILL,
1052                                   "pps_info_size   %d\n", pps_info_size);
1053                         vpu_debug(DEBUG_PPS_FILL,
1054                                   "scaling_list_addr_offset %d\n",
1055                                   scaling_list_addr_offset);
1056
1057                         if (pps_info_count) {
1058                                 char *pps = (char *)ion_map_kernel(
1059                                                 pservice->ion_client, hdl);
1060                                 vpu_debug(DEBUG_PPS_FILL,
1061                                           "scaling list setting pps %p\n", pps);
1062                                 pps += pps_info_offset;
1063
1064                                 fill_scaling_list_addr_in_pps(
1065                                                 data, reg, pps,
1066                                                 pps_info_count,
1067                                                 pps_info_size,
1068                                                 scaling_list_addr_offset);
1069                         }
1070                 }
1071
1072                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1073
1074                 if (!mem_region) {
1075                         ion_free(pservice->ion_client, hdl);
1076                         return -1;
1077                 }
1078
1079                 mem_region->hdl = hdl;
1080                 mem_region->reg_idx = tbl[i];
1081
1082                 if (data->mmu_dev)
1083                         ret = ion_map_iommu(data->dev,
1084                                             pservice->ion_client,
1085                                             mem_region->hdl,
1086                                             &mem_region->iova,
1087                                             &mem_region->len);
1088                 else
1089                         ret = ion_phys(pservice->ion_client,
1090                                        mem_region->hdl,
1091                                        (ion_phys_addr_t *)&mem_region->iova,
1092                                        (size_t *)&mem_region->len);
1093
1094                 if (ret < 0) {
1095                         dev_err(pservice->dev, "reg %d fd %d ion map iommu failed\n",
1096                                 tbl[i], usr_fd);
1097                         kfree(mem_region);
1098                         ion_free(pservice->ion_client, hdl);
1099                         return ret;
1100                 }
1101
1102                 /*
1103                  * special for vpu dec num 12: record decoded length
1104                  * hacking for decoded length
1105                  * NOTE: not a perfect fix, the fd is not recorded
1106                  */
1107                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1108                         reg->dec_base = mem_region->iova + offset;
1109                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1110                                   reg->dec_base);
1111                 }
1112
1113                 reg->reg[tbl[i]] = mem_region->iova + offset;
1114                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1115                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1116         }
1117
1118         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1119                 for (i = 0; i < ext_inf->cnt; i++) {
1120                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1121                                   ext_inf->elem[i].index,
1122                                   ext_inf->elem[i].offset);
1123                         reg->reg[ext_inf->elem[i].index] +=
1124                                 ext_inf->elem[i].offset;
1125                 }
1126         }
1127
1128         return 0;
1129 }
1130
1131 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1132                                         struct vpu_reg *reg,
1133                                         struct extra_info_for_iommu *ext_inf)
1134 {
1135         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1136
1137         if (type < FMT_TYPE_BUTT) {
1138                 const struct vpu_trans_info *info = &reg->trans[type];
1139                 const u8 *tbl = info->table;
1140                 int size = info->count;
1141
1142                 return vcodec_bufid_to_iova(data, tbl, size, reg, ext_inf);
1143         }
1144         pr_err("found invalid format type!\n");
1145         return -1;
1146 }
1147
1148 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1149 {
1150
1151         if (!soc_is_rk2928g()) {
1152                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1153                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1154                                 if (reg_probe_width(reg) > 3200) {
1155                                         /*raise frequency for 4k avc.*/
1156                                         reg->freq = VPU_FREQ_600M;
1157                                 }
1158                         } else {
1159                                 if (reg_check_interlace(reg))
1160                                         reg->freq = VPU_FREQ_400M;
1161                         }
1162                 }
1163                 if (data->hw_id == HEVC_ID) {
1164                         if (reg_probe_hevc_y_stride(reg) > 60000)
1165                                 reg->freq = VPU_FREQ_400M;
1166                 }
1167                 if (reg->type == VPU_PP)
1168                         reg->freq = VPU_FREQ_400M;
1169         }
1170 }
1171
1172 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1173                                 struct vpu_session *session,
1174                                 void __user *src, u32 size)
1175 {
1176         struct vpu_service_info *pservice = data->pservice;
1177         int extra_size = 0;
1178         struct extra_info_for_iommu extra_info;
1179         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1180                                       GFP_KERNEL);
1181
1182         vpu_debug_enter();
1183
1184         if (NULL == reg) {
1185                 vpu_err("error: kmalloc fail in reg_init\n");
1186                 return NULL;
1187         }
1188
1189         if (size > data->reg_size) {
1190                 pr_err("vpu reg size %u is larger than hw reg size %u\n",
1191                        size, data->reg_size);
1192                 extra_size = size - data->reg_size;
1193                 size = data->reg_size;
1194         }
1195         reg->session = session;
1196         reg->data = data;
1197         reg->type = session->type;
1198         reg->size = size;
1199         reg->freq = VPU_FREQ_DEFAULT;
1200         reg->task = &data->task_info[session->type];
1201         reg->trans = data->trans_info;
1202         reg->reg = (u32 *)&reg[1];
1203         INIT_LIST_HEAD(&reg->session_link);
1204         INIT_LIST_HEAD(&reg->status_link);
1205
1206         INIT_LIST_HEAD(&reg->mem_region_list);
1207
1208         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1209                 vpu_err("error: copy_from_user failed in reg_init\n");
1210                 kfree(reg);
1211                 return NULL;
1212         }
1213
1214         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1215                 vpu_err("error: copy_from_user failed in reg_init\n");
1216                 kfree(reg);
1217                 return NULL;
1218         }
1219
1220         if (0 > vcodec_reg_address_translate(data, reg, &extra_info)) {
1221                 int i = 0;
1222
1223                 vpu_err("error: translate reg address failed, dumping regs\n");
1224                 for (i = 0; i < size >> 2; i++)
1225                         pr_err("reg[%02d]: %08x\n", i, *((u32 *)src + i));
1226
1227                 kfree(reg);
1228                 return NULL;
1229         }
1230
1231         mutex_lock(&pservice->lock);
1232         list_add_tail(&reg->status_link, &pservice->waiting);
1233         list_add_tail(&reg->session_link, &session->waiting);
1234         mutex_unlock(&pservice->lock);
1235
1236         if (pservice->auto_freq)
1237                 get_reg_freq(data, reg);
1238
1239         vpu_debug_leave();
1240         return reg;
1241 }
1242
1243 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1244 {
1245         struct vpu_service_info *pservice = data->pservice;
1246         struct vcodec_mem_region *mem_region = NULL, *n;
1247
1248         list_del_init(&reg->session_link);
1249         list_del_init(&reg->status_link);
1250         if (reg == pservice->reg_codec)
1251                 pservice->reg_codec = NULL;
1252         if (reg == pservice->reg_pproc)
1253                 pservice->reg_pproc = NULL;
1254
1255         /* release memory region attach to this registers table. */
1256         list_for_each_entry_safe(mem_region, n,
1257                         &reg->mem_region_list, reg_lnk) {
1258                 ion_free(pservice->ion_client, mem_region->hdl);
1259                 list_del_init(&mem_region->reg_lnk);
1260                 kfree(mem_region);
1261         }
1262
1263         kfree(reg);
1264 }
1265
1266 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1267                                  struct vpu_reg *reg)
1268 {
1269         vpu_debug_enter();
1270         list_del_init(&reg->status_link);
1271         list_add_tail(&reg->status_link, &pservice->running);
1272
1273         list_del_init(&reg->session_link);
1274         list_add_tail(&reg->session_link, &reg->session->running);
1275         vpu_debug_leave();
1276 }
1277
1278 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1279 {
1280         int i;
1281         u32 *dst = reg->reg;
1282
1283         vpu_debug_enter();
1284         for (i = 0; i < count; i++, src++)
1285                 *dst++ = readl_relaxed(src);
1286
1287         dst = (u32 *)&reg->reg[0];
1288         for (i = 0; i < count; i++)
1289                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1290
1291         vpu_debug_leave();
1292 }
1293
1294 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1295                                  struct vpu_reg *reg)
1296 {
1297         struct vpu_service_info *pservice = data->pservice;
1298         struct vpu_hw_info *hw_info = data->hw_info;
1299         struct vpu_task_info *task = reg->task;
1300
1301         vpu_debug_enter();
1302
1303         list_del_init(&reg->status_link);
1304         list_add_tail(&reg->status_link, &pservice->done);
1305
1306         list_del_init(&reg->session_link);
1307         list_add_tail(&reg->session_link, &reg->session->done);
1308
1309         switch (reg->type) {
1310         case VPU_ENC: {
1311                 pservice->reg_codec = NULL;
1312                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1313                 reg->reg[task->reg_irq] = pservice->irq_status;
1314         } break;
1315         case VPU_DEC: {
1316                 pservice->reg_codec = NULL;
1317                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1318
1319                 /* revert hack for decoded length */
1320                 if (task->reg_len > 0) {
1321                         int reg_len = task->reg_len;
1322                         u32 dec_get = reg->reg[reg_len];
1323                         s32 dec_length = dec_get - reg->dec_base;
1324
1325                         vpu_debug(DEBUG_REGISTER,
1326                                   "dec_get %08x dec_length %d\n",
1327                                   dec_get, dec_length);
1328                         reg->reg[reg_len] = dec_length << 10;
1329                 }
1330
1331                 reg->reg[task->reg_irq] = pservice->irq_status;
1332         } break;
1333         case VPU_PP: {
1334                 pservice->reg_pproc = NULL;
1335                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1336                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1337         } break;
1338         case VPU_DEC_PP: {
1339                 u32 pipe_mode;
1340                 u32 *regs = data->dec_dev.regs;
1341
1342                 pservice->reg_codec = NULL;
1343                 pservice->reg_pproc = NULL;
1344
1345                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1346
1347                 /* NOTE: remove pp pipeline mode flag first */
1348                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1349                 pipe_mode &= ~task->pipe_mask;
1350                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1351
1352                 /* revert hack for decoded length */
1353                 if (task->reg_len > 0) {
1354                         int reg_len = task->reg_len;
1355                         u32 dec_get = reg->reg[reg_len];
1356                         s32 dec_length = dec_get - reg->dec_base;
1357
1358                         vpu_debug(DEBUG_REGISTER,
1359                                   "dec_get %08x dec_length %d\n",
1360                                   dec_get, dec_length);
1361                         reg->reg[reg_len] = dec_length << 10;
1362                 }
1363
1364                 reg->reg[task->reg_irq] = pservice->irq_status;
1365         } break;
1366         default: {
1367                 vpu_err("error: copy reg from hw with unknown type %d\n",
1368                         reg->type);
1369         } break;
1370         }
1371         vcodec_exit_mode(data);
1372
1373         atomic_sub(1, &reg->session->task_running);
1374         atomic_sub(1, &pservice->total_running);
1375         wake_up(&reg->session->wait);
1376
1377         vpu_debug_leave();
1378 }
1379
1380 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1381                                  struct vpu_reg *reg)
1382 {
1383         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1384
1385         if (curr == reg->freq)
1386                 return;
1387
1388         atomic_set(&pservice->freq_status, reg->freq);
1389         switch (reg->freq) {
1390         case VPU_FREQ_200M: {
1391                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1392         } break;
1393         case VPU_FREQ_266M: {
1394                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1395         } break;
1396         case VPU_FREQ_300M: {
1397                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1398         } break;
1399         case VPU_FREQ_400M: {
1400                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1401         } break;
1402         case VPU_FREQ_500M: {
1403                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1404         } break;
1405         case VPU_FREQ_600M: {
1406                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1407         } break;
1408         default: {
1409                 unsigned long rate = 300*MHZ;
1410
1411                 if (soc_is_rk2928g())
1412                         rate = 400*MHZ;
1413
1414                 clk_set_rate(pservice->aclk_vcodec, rate);
1415         } break;
1416         }
1417 }
1418
1419 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1420 {
1421         struct vpu_service_info *pservice = data->pservice;
1422         struct vpu_task_info *task = reg->task;
1423         struct vpu_hw_info *hw_info = data->hw_info;
1424         int i;
1425         u32 *src = (u32 *)&reg->reg[0];
1426         u32 enable_mask = task->enable_mask;
1427         u32 gating_mask = task->gating_mask;
1428         u32 reg_en = task->reg_en;
1429
1430         vpu_debug_enter();
1431
1432         atomic_add(1, &pservice->total_running);
1433         atomic_add(1, &reg->session->task_running);
1434
1435         if (pservice->auto_freq)
1436                 vpu_service_set_freq(pservice, reg);
1437
1438         vcodec_enter_mode(data);
1439
1440         switch (reg->type) {
1441         case VPU_ENC: {
1442                 u32 *dst = data->enc_dev.regs;
1443                 u32 base = 0;
1444                 u32 end  = hw_info->enc_reg_num;
1445                 /* u32 reg_gating = task->reg_gating; */
1446
1447                 pservice->reg_codec = reg;
1448
1449                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1450                           base, end, reg_en, enable_mask, gating_mask);
1451
1452                 VEPU_CLEAN_CACHE(dst);
1453
1454                 if (debug & DEBUG_SET_REG)
1455                         for (i = base; i < end; i++)
1456                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1457                                           i, src[i]);
1458
1459                 /*
1460                  * NOTE: encoder need to setup mode first
1461                  */
1462                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1463
1464                 /* NOTE: encoder gating is not on enable register */
1465                 /* src[reg_gating] |= gating_mask; */
1466
1467                 for (i = base; i < end; i++) {
1468                         if (i != reg_en)
1469                                 writel_relaxed(src[i], dst + i);
1470                 }
1471
1472                 writel(src[reg_en], dst + reg_en);
1473                 dsb(sy);
1474
1475                 time_record(reg->task, 0);
1476         } break;
1477         case VPU_DEC: {
1478                 u32 *dst = data->dec_dev.regs;
1479                 u32 len = hw_info->dec_reg_num;
1480                 u32 base = hw_info->base_dec;
1481                 u32 end  = hw_info->end_dec;
1482
1483                 pservice->reg_codec = reg;
1484
1485                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1486                           base, end, reg_en, enable_mask, gating_mask);
1487
1488                 VDPU_CLEAN_CACHE(dst);
1489
1490                 /* on rkvdec set cache size to 64byte */
1491                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1492                         u32 *cache_base = dst + 0x100;
1493                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1494                         writel_relaxed(val, cache_base + 0x07);
1495                         writel_relaxed(val, cache_base + 0x17);
1496                 }
1497
1498                 if (debug & DEBUG_SET_REG)
1499                         for (i = 0; i < len; i++)
1500                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1501                                           i, src[i]);
1502
1503                 /*
1504                  * NOTE: The end register is invalid. Do NOT write to it
1505                  *       Also the base register must be written
1506                  */
1507                 for (i = base; i < end; i++) {
1508                         if (i != reg_en)
1509                                 writel_relaxed(src[i], dst + i);
1510                 }
1511
1512                 writel(src[reg_en] | gating_mask, dst + reg_en);
1513                 dsb(sy);
1514
1515                 time_record(reg->task, 0);
1516         } break;
1517         case VPU_PP: {
1518                 u32 *dst = data->dec_dev.regs;
1519                 u32 base = hw_info->base_pp;
1520                 u32 end  = hw_info->end_pp;
1521
1522                 pservice->reg_pproc = reg;
1523
1524                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1525                           base, end, reg_en, enable_mask, gating_mask);
1526
1527                 if (debug & DEBUG_SET_REG)
1528                         for (i = base; i < end; i++)
1529                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1530                                           i, src[i]);
1531
1532                 for (i = base; i < end; i++) {
1533                         if (i != reg_en)
1534                                 writel_relaxed(src[i], dst + i);
1535                 }
1536
1537                 writel(src[reg_en] | gating_mask, dst + reg_en);
1538                 dsb(sy);
1539
1540                 time_record(reg->task, 0);
1541         } break;
1542         case VPU_DEC_PP: {
1543                 u32 *dst = data->dec_dev.regs;
1544                 u32 base = hw_info->base_dec_pp;
1545                 u32 end  = hw_info->end_dec_pp;
1546
1547                 pservice->reg_codec = reg;
1548                 pservice->reg_pproc = reg;
1549
1550                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1551                           base, end, reg_en, enable_mask, gating_mask);
1552
1553                 /* VDPU_SOFT_RESET(dst); */
1554                 VDPU_CLEAN_CACHE(dst);
1555
1556                 if (debug & DEBUG_SET_REG)
1557                         for (i = base; i < end; i++)
1558                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1559                                           i, src[i]);
1560
1561                 for (i = base; i < end; i++) {
1562                         if (i != reg_en)
1563                                 writel_relaxed(src[i], dst + i);
1564                 }
1565
1566                 /* NOTE: dec output must be disabled */
1567
1568                 writel(src[reg_en] | gating_mask, dst + reg_en);
1569                 dsb(sy);
1570
1571                 time_record(reg->task, 0);
1572         } break;
1573         default: {
1574                 vpu_err("error: unsupport session type %d", reg->type);
1575                 atomic_sub(1, &pservice->total_running);
1576                 atomic_sub(1, &reg->session->task_running);
1577         } break;
1578         }
1579
1580         vpu_debug_leave();
1581 }
1582
1583 static void try_set_reg(struct vpu_subdev_data *data)
1584 {
1585         struct vpu_service_info *pservice = data->pservice;
1586
1587         vpu_debug_enter();
1588         if (!list_empty(&pservice->waiting)) {
1589                 struct vpu_reg *reg_codec = pservice->reg_codec;
1590                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1591                 int can_set = 0;
1592                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1593                 int reset_request = atomic_read(&pservice->reset_request);
1594                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1595                                 struct vpu_reg, status_link);
1596
1597                 vpu_service_power_on(pservice);
1598
1599                 if (change_able || !reset_request) {
1600                         switch (reg->type) {
1601                         case VPU_ENC: {
1602                                 if (change_able)
1603                                         can_set = 1;
1604                         } break;
1605                         case VPU_DEC: {
1606                                 if (reg_codec == NULL)
1607                                         can_set = 1;
1608                                 if (pservice->auto_freq && (reg_pproc != NULL))
1609                                         can_set = 0;
1610                         } break;
1611                         case VPU_PP: {
1612                                 if (reg_codec == NULL) {
1613                                         if (reg_pproc == NULL)
1614                                                 can_set = 1;
1615                                 } else {
1616                                         if ((reg_codec->type == VPU_DEC) &&
1617                                             (reg_pproc == NULL))
1618                                                 can_set = 1;
1619
1620                                         /*
1621                                          * NOTE:
1622                                          * can not charge frequency
1623                                          * when vpu is working
1624                                          */
1625                                         if (pservice->auto_freq)
1626                                                 can_set = 0;
1627                                 }
1628                         } break;
1629                         case VPU_DEC_PP: {
1630                                 if (change_able)
1631                                         can_set = 1;
1632                                 } break;
1633                         default: {
1634                                 pr_err("undefined reg type %d\n", reg->type);
1635                         } break;
1636                         }
1637                 }
1638
1639                 /* then check reset request */
1640                 if (reset_request && !change_able)
1641                         reset_request = 0;
1642
1643                 /* do reset before setting registers */
1644                 if (reset_request)
1645                         vpu_reset(data);
1646
1647                 if (can_set) {
1648                         reg_from_wait_to_run(pservice, reg);
1649                         reg_copy_to_hw(reg->data, reg);
1650                 }
1651         }
1652         vpu_debug_leave();
1653 }
1654
1655 static int return_reg(struct vpu_subdev_data *data,
1656                       struct vpu_reg *reg, u32 __user *dst)
1657 {
1658         struct vpu_hw_info *hw_info = data->hw_info;
1659         size_t size = reg->size;
1660         u32 base;
1661
1662         vpu_debug_enter();
1663         switch (reg->type) {
1664         case VPU_ENC: {
1665                 base = 0;
1666         } break;
1667         case VPU_DEC: {
1668                 base = hw_info->base_dec_pp;
1669         } break;
1670         case VPU_PP: {
1671                 base = hw_info->base_pp;
1672         } break;
1673         case VPU_DEC_PP: {
1674                 base = hw_info->base_dec_pp;
1675         } break;
1676         default: {
1677                 vpu_err("error: copy reg to user with unknown type %d\n",
1678                         reg->type);
1679                 return -EFAULT;
1680         } break;
1681         }
1682
1683         if (copy_to_user(dst, &reg->reg[base], size)) {
1684                 vpu_err("error: return_reg copy_to_user failed\n");
1685                 return -EFAULT;
1686         }
1687
1688         reg_deinit(data, reg);
1689         vpu_debug_leave();
1690         return 0;
1691 }
1692
1693 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1694                               unsigned long arg)
1695 {
1696         struct vpu_subdev_data *data =
1697                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1698                              struct vpu_subdev_data, cdev);
1699         struct vpu_service_info *pservice = data->pservice;
1700         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1701
1702         vpu_debug_enter();
1703         if (NULL == session)
1704                 return -EINVAL;
1705
1706         switch (cmd) {
1707         case VPU_IOC_SET_CLIENT_TYPE: {
1708                 session->type = (enum VPU_CLIENT_TYPE)arg;
1709                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1710                           session->pid, session->type);
1711         } break;
1712         case VPU_IOC_GET_HW_FUSE_STATUS: {
1713                 struct vpu_request req;
1714
1715                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1716                           session->pid, session->type);
1717                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1718                         vpu_err("error: get hw status copy_from_user failed\n");
1719                         return -EFAULT;
1720                 } else {
1721                         void *config = (session->type != VPU_ENC) ?
1722                                        ((void *)&pservice->dec_config) :
1723                                        ((void *)&pservice->enc_config);
1724                         size_t size = (session->type != VPU_ENC) ?
1725                                       (sizeof(struct vpu_dec_config)) :
1726                                       (sizeof(struct vpu_enc_config));
1727                         if (copy_to_user((void __user *)req.req,
1728                                          config, size)) {
1729                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1730                                         session->type);
1731                                 return -EFAULT;
1732                         }
1733                 }
1734         } break;
1735         case VPU_IOC_SET_REG: {
1736                 struct vpu_request req;
1737                 struct vpu_reg *reg;
1738
1739                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1740                           session->pid, session->type);
1741                 if (copy_from_user(&req, (void __user *)arg,
1742                                    sizeof(struct vpu_request))) {
1743                         vpu_err("error: set reg copy_from_user failed\n");
1744                         return -EFAULT;
1745                 }
1746                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1747                 if (NULL == reg) {
1748                         return -EFAULT;
1749                 } else {
1750                         mutex_lock(&pservice->lock);
1751                         try_set_reg(data);
1752                         mutex_unlock(&pservice->lock);
1753                 }
1754         } break;
1755         case VPU_IOC_GET_REG: {
1756                 struct vpu_request req;
1757                 struct vpu_reg *reg;
1758                 int ret;
1759
1760                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1761                           session->pid, session->type);
1762                 if (copy_from_user(&req, (void __user *)arg,
1763                                    sizeof(struct vpu_request))) {
1764                         vpu_err("error: get reg copy_from_user failed\n");
1765                         return -EFAULT;
1766                 }
1767
1768                 ret = wait_event_timeout(session->wait,
1769                                          !list_empty(&session->done),
1770                                          VPU_TIMEOUT_DELAY);
1771
1772                 if (!list_empty(&session->done)) {
1773                         if (ret < 0)
1774                                 vpu_err("warning: pid %d wait task error ret %d\n",
1775                                         session->pid, ret);
1776                         ret = 0;
1777                 } else {
1778                         if (unlikely(ret < 0)) {
1779                                 vpu_err("error: pid %d wait task ret %d\n",
1780                                         session->pid, ret);
1781                         } else if (ret == 0) {
1782                                 vpu_err("error: pid %d wait %d task done timeout\n",
1783                                         session->pid,
1784                                         atomic_read(&session->task_running));
1785                                 ret = -ETIMEDOUT;
1786                         }
1787                 }
1788
1789                 if (ret < 0) {
1790                         int task_running = atomic_read(&session->task_running);
1791
1792                         mutex_lock(&pservice->lock);
1793                         vpu_service_dump(pservice);
1794                         if (task_running) {
1795                                 atomic_set(&session->task_running, 0);
1796                                 atomic_sub(task_running,
1797                                            &pservice->total_running);
1798                                 pr_err("%d task is running but not return, reset hardware...",
1799                                        task_running);
1800                                 vpu_reset(data);
1801                                 pr_err("done\n");
1802                         }
1803                         vpu_service_session_clear(data, session);
1804                         mutex_unlock(&pservice->lock);
1805                         return ret;
1806                 }
1807
1808                 mutex_lock(&pservice->lock);
1809                 reg = list_entry(session->done.next,
1810                                  struct vpu_reg, session_link);
1811                 return_reg(data, reg, (u32 __user *)req.req);
1812                 mutex_unlock(&pservice->lock);
1813         } break;
1814         case VPU_IOC_PROBE_IOMMU_STATUS: {
1815                 int iommu_enable = 1;
1816
1817                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1818
1819                 if (copy_to_user((void __user *)arg,
1820                                  &iommu_enable, sizeof(int))) {
1821                         vpu_err("error: iommu status copy_to_user failed\n");
1822                         return -EFAULT;
1823                 }
1824         } break;
1825         default: {
1826                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1827         } break;
1828         }
1829         vpu_debug_leave();
1830         return 0;
1831 }
1832
1833 #ifdef CONFIG_COMPAT
1834 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1835                                      unsigned long arg)
1836 {
1837         struct vpu_subdev_data *data =
1838                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1839                              struct vpu_subdev_data, cdev);
1840         struct vpu_service_info *pservice = data->pservice;
1841         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1842
1843         vpu_debug_enter();
1844         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1845                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1846         if (NULL == session)
1847                 return -EINVAL;
1848
1849         switch (cmd) {
1850         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1851                 session->type = (enum VPU_CLIENT_TYPE)arg;
1852                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1853                           session->type);
1854         } break;
1855         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1856                 struct compat_vpu_request req;
1857
1858                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1859                           session->type);
1860                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1861                                    sizeof(struct compat_vpu_request))) {
1862                         vpu_err("error: compat get hw status copy_from_user failed\n");
1863                         return -EFAULT;
1864                 } else {
1865                         void *config = (session->type != VPU_ENC) ?
1866                                        ((void *)&pservice->dec_config) :
1867                                        ((void *)&pservice->enc_config);
1868                         size_t size = (session->type != VPU_ENC) ?
1869                                       (sizeof(struct vpu_dec_config)) :
1870                                       (sizeof(struct vpu_enc_config));
1871
1872                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1873                                          config, size)) {
1874                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1875                                         session->type);
1876                                 return -EFAULT;
1877                         }
1878                 }
1879         } break;
1880         case COMPAT_VPU_IOC_SET_REG: {
1881                 struct compat_vpu_request req;
1882                 struct vpu_reg *reg;
1883
1884                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1885                           session->type);
1886                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1887                                    sizeof(struct compat_vpu_request))) {
1888                         vpu_err("compat set_reg copy_from_user failed\n");
1889                         return -EFAULT;
1890                 }
1891                 reg = reg_init(data, session,
1892                                compat_ptr((compat_uptr_t)req.req), req.size);
1893                 if (NULL == reg) {
1894                         return -EFAULT;
1895                 } else {
1896                         mutex_lock(&pservice->lock);
1897                         try_set_reg(data);
1898                         mutex_unlock(&pservice->lock);
1899                 }
1900         } break;
1901         case COMPAT_VPU_IOC_GET_REG: {
1902                 struct compat_vpu_request req;
1903                 struct vpu_reg *reg;
1904                 int ret;
1905
1906                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1907                           session->type);
1908                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1909                                    sizeof(struct compat_vpu_request))) {
1910                         vpu_err("compat get reg copy_from_user failed\n");
1911                         return -EFAULT;
1912                 }
1913
1914                 ret = wait_event_timeout(session->wait,
1915                                          !list_empty(&session->done),
1916                                          VPU_TIMEOUT_DELAY);
1917
1918                 if (!list_empty(&session->done)) {
1919                         if (ret < 0)
1920                                 vpu_err("warning: pid %d wait task error ret %d\n",
1921                                         session->pid, ret);
1922                         ret = 0;
1923                 } else {
1924                         if (unlikely(ret < 0)) {
1925                                 vpu_err("error: pid %d wait task ret %d\n",
1926                                         session->pid, ret);
1927                         } else if (ret == 0) {
1928                                 vpu_err("error: pid %d wait %d task done timeout\n",
1929                                         session->pid,
1930                                         atomic_read(&session->task_running));
1931                                 ret = -ETIMEDOUT;
1932                         }
1933                 }
1934
1935                 if (ret < 0) {
1936                         int task_running = atomic_read(&session->task_running);
1937
1938                         mutex_lock(&pservice->lock);
1939                         vpu_service_dump(pservice);
1940                         if (task_running) {
1941                                 atomic_set(&session->task_running, 0);
1942                                 atomic_sub(task_running,
1943                                            &pservice->total_running);
1944                                 pr_err("%d task is running but not return, reset hardware...",
1945                                        task_running);
1946                                 vpu_reset(data);
1947                                 pr_err("done\n");
1948                         }
1949                         vpu_service_session_clear(data, session);
1950                         mutex_unlock(&pservice->lock);
1951                         return ret;
1952                 }
1953
1954                 mutex_lock(&pservice->lock);
1955                 reg = list_entry(session->done.next,
1956                                  struct vpu_reg, session_link);
1957                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
1958                 mutex_unlock(&pservice->lock);
1959         } break;
1960         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
1961                 int iommu_enable = 1;
1962
1963                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
1964
1965                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
1966                                  &iommu_enable, sizeof(int))) {
1967                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1968                         return -EFAULT;
1969                 }
1970         } break;
1971         default: {
1972                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1973         } break;
1974         }
1975         vpu_debug_leave();
1976         return 0;
1977 }
1978 #endif
1979
1980 static int vpu_service_check_hw(struct vpu_subdev_data *data)
1981 {
1982         int ret = -EINVAL, i = 0;
1983         u32 hw_id = readl_relaxed(data->regs);
1984
1985         hw_id = (hw_id >> 16) & 0xFFFF;
1986         pr_info("checking hw id %x\n", hw_id);
1987         data->hw_info = NULL;
1988         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
1989                 struct vcodec_info *info = &vcodec_info_set[i];
1990
1991                 if (hw_id == info->hw_id) {
1992                         data->hw_id = info->hw_id;
1993                         data->hw_info = info->hw_info;
1994                         data->task_info = info->task_info;
1995                         data->trans_info = info->trans_info;
1996                         ret = 0;
1997                         break;
1998                 }
1999         }
2000         return ret;
2001 }
2002
2003 static int vpu_service_open(struct inode *inode, struct file *filp)
2004 {
2005         struct vpu_subdev_data *data = container_of(
2006                         inode->i_cdev, struct vpu_subdev_data, cdev);
2007         struct vpu_service_info *pservice = data->pservice;
2008         struct vpu_session *session = kmalloc(sizeof(*session), GFP_KERNEL);
2009
2010         vpu_debug_enter();
2011
2012         if (NULL == session) {
2013                 vpu_err("error: unable to allocate memory for vpu_session.");
2014                 return -ENOMEM;
2015         }
2016
2017         session->type   = VPU_TYPE_BUTT;
2018         session->pid    = current->pid;
2019         INIT_LIST_HEAD(&session->waiting);
2020         INIT_LIST_HEAD(&session->running);
2021         INIT_LIST_HEAD(&session->done);
2022         INIT_LIST_HEAD(&session->list_session);
2023         init_waitqueue_head(&session->wait);
2024         atomic_set(&session->task_running, 0);
2025         mutex_lock(&pservice->lock);
2026         list_add_tail(&session->list_session, &pservice->session);
2027         filp->private_data = (void *)session;
2028         mutex_unlock(&pservice->lock);
2029
2030         pr_debug("dev opened\n");
2031         vpu_debug_leave();
2032         return nonseekable_open(inode, filp);
2033 }
2034
2035 static int vpu_service_release(struct inode *inode, struct file *filp)
2036 {
2037         struct vpu_subdev_data *data = container_of(
2038                         inode->i_cdev, struct vpu_subdev_data, cdev);
2039         struct vpu_service_info *pservice = data->pservice;
2040         int task_running;
2041         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2042
2043         vpu_debug_enter();
2044         if (NULL == session)
2045                 return -EINVAL;
2046
2047         task_running = atomic_read(&session->task_running);
2048         if (task_running) {
2049                 pr_err("error: session %d still has %d task running when closing\n",
2050                        session->pid, task_running);
2051                 msleep(50);
2052         }
2053         wake_up(&session->wait);
2054
2055         mutex_lock(&pservice->lock);
2056         /* remove this filp from the asynchronusly notified filp's */
2057         list_del_init(&session->list_session);
2058         vpu_service_session_clear(data, session);
2059         kfree(session);
2060         filp->private_data = NULL;
2061         mutex_unlock(&pservice->lock);
2062
2063         pr_debug("dev closed\n");
2064         vpu_debug_leave();
2065         return 0;
2066 }
2067
2068 static const struct file_operations vpu_service_fops = {
2069         .unlocked_ioctl = vpu_service_ioctl,
2070         .open           = vpu_service_open,
2071         .release        = vpu_service_release,
2072 #ifdef CONFIG_COMPAT
2073         .compat_ioctl   = compat_vpu_service_ioctl,
2074 #endif
2075 };
2076
2077 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2078 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2079 static irqreturn_t vepu_irq(int irq, void *dev_id);
2080 static irqreturn_t vepu_isr(int irq, void *dev_id);
2081 static void get_hw_info(struct vpu_subdev_data *data);
2082
2083 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2084 {
2085         struct device_node *dn = NULL;
2086         struct platform_device *pd = NULL;
2087         struct device *ret = NULL;
2088
2089         dn = of_find_compatible_node(NULL, NULL, compt);
2090         if (!dn) {
2091                 pr_err("can't find device node %s \r\n", compt);
2092                 return NULL;
2093         }
2094
2095         pd = of_find_device_by_node(dn);
2096         if (!pd) {
2097                 pr_err("can't find platform device in device node %s\n", compt);
2098                 return  NULL;
2099         }
2100         ret = &pd->dev;
2101
2102         return ret;
2103 }
2104
2105 #ifdef CONFIG_IOMMU_API
2106 static inline void platform_set_sysmmu(struct device *iommu,
2107                                        struct device *dev)
2108 {
2109         dev->archdata.iommu = iommu;
2110 }
2111 #else
2112 static inline void platform_set_sysmmu(struct device *iommu,
2113                                        struct device *dev)
2114 {
2115 }
2116 #endif
2117
2118 int vcodec_sysmmu_fault_hdl(struct device *dev,
2119                             enum rk_iommu_inttype itype,
2120                             unsigned long pgtable_base,
2121                             unsigned long fault_addr, unsigned int status)
2122 {
2123         struct platform_device *pdev;
2124         struct vpu_service_info *pservice;
2125         struct vpu_subdev_data *data;
2126
2127         vpu_debug_enter();
2128
2129         if (dev == NULL) {
2130                 pr_err("invalid NULL dev\n");
2131                 return 0;
2132         }
2133
2134         pdev = container_of(dev, struct platform_device, dev);
2135         if (pdev == NULL) {
2136                 pr_err("invalid NULL platform_device\n");
2137                 return 0;
2138         }
2139
2140         data = platform_get_drvdata(pdev);
2141         if (data == NULL) {
2142                 pr_err("invalid NULL vpu_subdev_data\n");
2143                 return 0;
2144         }
2145
2146         pservice = data->pservice;
2147         if (pservice == NULL) {
2148                 pr_err("invalid NULL vpu_service_info\n");
2149                 return 0;
2150         }
2151
2152         if (pservice->reg_codec) {
2153                 struct vpu_reg *reg = pservice->reg_codec;
2154                 struct vcodec_mem_region *mem, *n;
2155                 int i = 0;
2156
2157                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2158                 if (!list_empty(&reg->mem_region_list)) {
2159                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2160                                                  reg_lnk) {
2161                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2162                                        mem->reg_idx, i, mem->iova, mem->len);
2163                                 i++;
2164                         }
2165                 } else {
2166                         pr_err("no memory region mapped\n");
2167                 }
2168
2169                 if (reg->data) {
2170                         struct vpu_subdev_data *data = reg->data;
2171                         u32 *base = (u32 *)data->dec_dev.regs;
2172                         u32 len = data->hw_info->dec_reg_num;
2173
2174                         pr_err("current errror register set:\n");
2175
2176                         for (i = 0; i < len; i++)
2177                                 pr_err("reg[%02d] %08x\n",
2178                                        i, readl_relaxed(base + i));
2179                 }
2180
2181                 pr_alert("vcodec, page fault occur, reset hw\n");
2182
2183                 /* reg->reg[101] = 1; */
2184                 vpu_reset(data);
2185         }
2186
2187         return 0;
2188 }
2189
2190 static int vcodec_subdev_probe(struct platform_device *pdev,
2191                                struct vpu_service_info *pservice)
2192 {
2193         int ret = 0;
2194         struct resource *res = NULL;
2195         u32 ioaddr = 0;
2196         u8 *regs = NULL;
2197         struct vpu_hw_info *hw_info = NULL;
2198         struct device *dev = &pdev->dev;
2199         char *name = (char *)dev_name(dev);
2200         struct device_node *np = pdev->dev.of_node;
2201         struct vpu_subdev_data *data =
2202                 devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2203         u32 iommu_en = 0;
2204         char mmu_dev_dts_name[40];
2205
2206         of_property_read_u32(np, "iommu_enabled", &iommu_en);
2207
2208         pr_info("probe device %s\n", dev_name(dev));
2209
2210         data->pservice = pservice;
2211         data->dev = dev;
2212
2213         of_property_read_string(np, "name", (const char **)&name);
2214         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2215
2216         if (pservice->reg_base == 0) {
2217                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2218                 data->regs = devm_ioremap_resource(dev, res);
2219                 if (IS_ERR(data->regs)) {
2220                         ret = PTR_ERR(data->regs);
2221                         goto err;
2222                 }
2223                 ioaddr = res->start;
2224         } else {
2225                 data->regs = pservice->reg_base;
2226                 ioaddr = pservice->ioaddr;
2227         }
2228
2229         clear_bit(MMU_ACTIVATED, &data->state);
2230         vcodec_enter_mode(data);
2231         ret = vpu_service_check_hw(data);
2232         if (ret < 0) {
2233                 vpu_err("error: hw info check faild\n");
2234                 goto err;
2235         }
2236
2237         hw_info = data->hw_info;
2238         regs = (u8 *)data->regs;
2239
2240         if (hw_info->dec_reg_num) {
2241                 data->dec_dev.iosize = hw_info->dec_io_size;
2242                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2243         }
2244
2245         if (hw_info->enc_reg_num) {
2246                 data->enc_dev.iosize = hw_info->enc_io_size;
2247                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2248         }
2249
2250         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2251
2252         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2253         if (data->irq_enc > 0) {
2254                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2255                                                 vepu_irq, vepu_isr,
2256                                                 IRQF_SHARED, dev_name(dev),
2257                                                 (void *)data);
2258                 if (ret) {
2259                         dev_err(dev, "error: can't request vepu irq %d\n",
2260                                 data->irq_enc);
2261                         goto err;
2262                 }
2263         }
2264         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2265         if (data->irq_dec > 0) {
2266                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2267                                                 vdpu_irq, vdpu_isr,
2268                                                 IRQF_SHARED, dev_name(dev),
2269                                                 (void *)data);
2270                 if (ret) {
2271                         dev_err(dev, "error: can't request vdpu irq %d\n",
2272                                 data->irq_dec);
2273                         goto err;
2274                 }
2275         }
2276         atomic_set(&data->dec_dev.irq_count_codec, 0);
2277         atomic_set(&data->dec_dev.irq_count_pp, 0);
2278         atomic_set(&data->enc_dev.irq_count_codec, 0);
2279         atomic_set(&data->enc_dev.irq_count_pp, 0);
2280
2281         if (iommu_en) {
2282                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
2283                         sprintf(mmu_dev_dts_name,
2284                                 HEVC_IOMMU_COMPATIBLE_NAME);
2285                 else if (data->mode == VCODEC_RUNNING_MODE_VPU)
2286                         sprintf(mmu_dev_dts_name,
2287                                 VPU_IOMMU_COMPATIBLE_NAME);
2288                 else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2289                         sprintf(mmu_dev_dts_name, VDEC_IOMMU_COMPATIBLE_NAME);
2290                 else
2291                         sprintf(mmu_dev_dts_name,
2292                                 HEVC_IOMMU_COMPATIBLE_NAME);
2293
2294                 data->mmu_dev =
2295                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2296
2297                 if (data->mmu_dev)
2298                         platform_set_sysmmu(data->mmu_dev, dev);
2299
2300                 rockchip_iovmm_set_fault_handler(dev, vcodec_sysmmu_fault_hdl);
2301         }
2302
2303         get_hw_info(data);
2304         pservice->auto_freq = true;
2305
2306         vcodec_exit_mode(data);
2307         /* create device node */
2308         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2309         if (ret) {
2310                 dev_err(dev, "alloc dev_t failed\n");
2311                 goto err;
2312         }
2313
2314         cdev_init(&data->cdev, &vpu_service_fops);
2315
2316         data->cdev.owner = THIS_MODULE;
2317         data->cdev.ops = &vpu_service_fops;
2318
2319         ret = cdev_add(&data->cdev, data->dev_t, 1);
2320
2321         if (ret) {
2322                 dev_err(dev, "add dev_t failed\n");
2323                 goto err;
2324         }
2325
2326         data->cls = class_create(THIS_MODULE, name);
2327
2328         if (IS_ERR(data->cls)) {
2329                 ret = PTR_ERR(data->cls);
2330                 dev_err(dev, "class_create err:%d\n", ret);
2331                 goto err;
2332         }
2333
2334         data->child_dev = device_create(data->cls, dev,
2335                 data->dev_t, NULL, name);
2336
2337         platform_set_drvdata(pdev, data);
2338
2339         INIT_LIST_HEAD(&data->lnk_service);
2340         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2341
2342 #ifdef CONFIG_DEBUG_FS
2343         data->debugfs_dir = vcodec_debugfs_create_device_dir(name, parent);
2344         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2345                 data->debugfs_file_regs =
2346                         debugfs_create_file("regs", 0664, data->debugfs_dir,
2347                                         data, &debug_vcodec_fops);
2348         else
2349                 vpu_err("create debugfs dir %s failed\n", name);
2350 #endif
2351         return 0;
2352 err:
2353         if (data->child_dev) {
2354                 device_destroy(data->cls, data->dev_t);
2355                 cdev_del(&data->cdev);
2356                 unregister_chrdev_region(data->dev_t, 1);
2357         }
2358
2359         if (data->cls)
2360                 class_destroy(data->cls);
2361         return -1;
2362 }
2363
2364 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2365 {
2366         struct vpu_service_info *pservice = data->pservice;
2367
2368         mutex_lock(&pservice->lock);
2369         cancel_delayed_work_sync(&pservice->power_off_work);
2370         vpu_service_power_off(pservice);
2371         mutex_unlock(&pservice->lock);
2372
2373         device_destroy(data->cls, data->dev_t);
2374         class_destroy(data->cls);
2375         cdev_del(&data->cdev);
2376         unregister_chrdev_region(data->dev_t, 1);
2377
2378 #ifdef CONFIG_DEBUG_FS
2379         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2380                 debugfs_remove_recursive(data->debugfs_dir);
2381 #endif
2382 }
2383
2384 static void vcodec_read_property(struct device_node *np,
2385                                  struct vpu_service_info *pservice)
2386 {
2387         pservice->mode_bit = 0;
2388         pservice->mode_ctrl = 0;
2389         pservice->subcnt = 0;
2390         pservice->grf_base = NULL;
2391
2392         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2393
2394         if (pservice->subcnt > 1) {
2395                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2396                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2397         }
2398 #ifdef CONFIG_MFD_SYSCON
2399         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2400         if (IS_ERR_OR_NULL(pservice->grf)) {
2401                 pservice->grf = NULL;
2402 #ifdef CONFIG_ARM
2403                 pservice->grf_base = RK_GRF_VIRT;
2404 #else
2405                 vpu_err("can't find vpu grf property\n");
2406                 return;
2407 #endif
2408         }
2409 #else
2410 #ifdef CONFIG_ARM
2411         pservice->grf_base = RK_GRF_VIRT;
2412 #else
2413         vpu_err("can't find vpu grf property\n");
2414         return;
2415 #endif
2416 #endif
2417
2418 #ifdef CONFIG_RESET_CONTROLLER
2419         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2420         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2421         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2422
2423         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2424                 pr_warn("No aclk reset resource define\n");
2425                 pservice->rst_a = NULL;
2426         }
2427
2428         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2429                 pr_warn("No hclk reset resource define\n");
2430                 pservice->rst_h = NULL;
2431         }
2432
2433         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2434                 pr_warn("No core reset resource define\n");
2435                 pservice->rst_v = NULL;
2436         }
2437 #endif
2438
2439         of_property_read_string(np, "name", (const char **)&pservice->name);
2440 }
2441
2442 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2443 {
2444         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2445         pservice->curr_mode = -1;
2446
2447         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2448         INIT_LIST_HEAD(&pservice->waiting);
2449         INIT_LIST_HEAD(&pservice->running);
2450         mutex_init(&pservice->lock);
2451
2452         INIT_LIST_HEAD(&pservice->done);
2453         INIT_LIST_HEAD(&pservice->session);
2454         INIT_LIST_HEAD(&pservice->subdev_list);
2455
2456         pservice->reg_pproc     = NULL;
2457         atomic_set(&pservice->total_running, 0);
2458         atomic_set(&pservice->enabled,       0);
2459         atomic_set(&pservice->power_on_cnt,  0);
2460         atomic_set(&pservice->power_off_cnt, 0);
2461         atomic_set(&pservice->reset_request, 0);
2462
2463         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2464
2465         pservice->ion_client = rockchip_ion_client_create("vpu");
2466         if (IS_ERR(pservice->ion_client)) {
2467                 vpu_err("failed to create ion client for vcodec ret %ld\n",
2468                         PTR_ERR(pservice->ion_client));
2469         } else {
2470                 vpu_debug(DEBUG_IOMMU, "vcodec ion client create success!\n");
2471         }
2472 }
2473
2474 static int vcodec_probe(struct platform_device *pdev)
2475 {
2476         int i;
2477         int ret = 0;
2478         struct resource *res = NULL;
2479         struct device *dev = &pdev->dev;
2480         struct device_node *np = pdev->dev.of_node;
2481         struct vpu_service_info *pservice =
2482                 devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
2483
2484         pservice->dev = dev;
2485
2486         vcodec_read_property(np, pservice);
2487         vcodec_init_drvdata(pservice);
2488
2489         if (strncmp(pservice->name, "hevc_service", 12) == 0)
2490                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2491         else if (strncmp(pservice->name, "vpu_service", 11) == 0)
2492                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2493         else if (strncmp(pservice->name, "rkvdec", 6) == 0)
2494                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2495         else
2496                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2497
2498         if (0 > vpu_get_clk(pservice))
2499                 goto err;
2500
2501         vpu_service_power_on(pservice);
2502
2503         if (of_property_read_bool(np, "reg")) {
2504                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2505
2506                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2507                 if (IS_ERR(pservice->reg_base)) {
2508                         vpu_err("ioremap registers base failed\n");
2509                         ret = PTR_ERR(pservice->reg_base);
2510                         goto err;
2511                 }
2512                 pservice->ioaddr = res->start;
2513         } else {
2514                 pservice->reg_base = 0;
2515         }
2516
2517         if (of_property_read_bool(np, "subcnt")) {
2518                 for (i = 0; i < pservice->subcnt; i++) {
2519                         struct device_node *sub_np;
2520                         struct platform_device *sub_pdev;
2521
2522                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2523                         sub_pdev = of_find_device_by_node(sub_np);
2524
2525                         vcodec_subdev_probe(sub_pdev, pservice);
2526                 }
2527         } else {
2528                 vcodec_subdev_probe(pdev, pservice);
2529         }
2530
2531         vpu_service_power_off(pservice);
2532
2533         pr_info("init success\n");
2534
2535         return 0;
2536
2537 err:
2538         pr_info("init failed\n");
2539         vpu_service_power_off(pservice);
2540         vpu_put_clk(pservice);
2541         wake_lock_destroy(&pservice->wake_lock);
2542
2543         return ret;
2544 }
2545
2546 static int vcodec_remove(struct platform_device *pdev)
2547 {
2548         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2549
2550         vcodec_subdev_remove(data);
2551         return 0;
2552 }
2553
2554 #if defined(CONFIG_OF)
2555 static const struct of_device_id vcodec_service_dt_ids[] = {
2556         {.compatible = "rockchip,vpu_service",},
2557         {.compatible = "rockchip,hevc_service",},
2558         {.compatible = "rockchip,vpu_combo",},
2559         {.compatible = "rockchip,rkvdec",},
2560         {},
2561 };
2562 #endif
2563
2564 static struct platform_driver vcodec_driver = {
2565         .probe = vcodec_probe,
2566         .remove = vcodec_remove,
2567         .driver = {
2568                 .name = "vcodec",
2569                 .owner = THIS_MODULE,
2570 #if defined(CONFIG_OF)
2571                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2572 #endif
2573         },
2574 };
2575
2576 static void get_hw_info(struct vpu_subdev_data *data)
2577 {
2578         struct vpu_service_info *pservice = data->pservice;
2579         struct vpu_dec_config *dec = &pservice->dec_config;
2580         struct vpu_enc_config *enc = &pservice->enc_config;
2581
2582         if (cpu_is_rk2928() || cpu_is_rk3036() ||
2583             cpu_is_rk30xx() || cpu_is_rk312x() ||
2584             cpu_is_rk3188())
2585                 dec->max_dec_pic_width = 1920;
2586         else
2587                 dec->max_dec_pic_width = 4096;
2588
2589         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2590                 dec->h264_support = 3;
2591                 dec->jpeg_support = 1;
2592                 dec->mpeg4_support = 2;
2593                 dec->vc1_support = 3;
2594                 dec->mpeg2_support = 1;
2595                 dec->pp_support = 1;
2596                 dec->sorenson_support = 1;
2597                 dec->ref_buf_support = 3;
2598                 dec->vp6_support = 1;
2599                 dec->vp7_support = 1;
2600                 dec->vp8_support = 1;
2601                 dec->avs_support = 1;
2602                 dec->jpeg_ext_support = 0;
2603                 dec->custom_mpeg4_support = 1;
2604                 dec->reserve = 0;
2605                 dec->mvc_support = 1;
2606
2607                 if (!cpu_is_rk3036()) {
2608                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2609
2610                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2611                         enc->h264_enabled = 1;
2612                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2613                         enc->jpeg_enabled = 1;
2614                         enc->vs_enabled = (config_reg >> 24) & 1;
2615                         enc->rgb_enabled = (config_reg >> 28) & 1;
2616                         enc->reg_size = data->reg_size;
2617                         enc->reserv[0] = 0;
2618                         enc->reserv[1] = 0;
2619                 }
2620
2621                 pservice->auto_freq = true;
2622                 vpu_debug(DEBUG_EXTRA_INFO, "vpu_service set to auto frequency mode\n");
2623                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2624
2625                 pservice->bug_dec_addr = cpu_is_rk30xx();
2626         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2627                 pservice->auto_freq = true;
2628                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2629         } else {
2630                 /* disable frequency switch in hevc.*/
2631                 pservice->auto_freq = false;
2632         }
2633 }
2634
2635 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2636 {
2637         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2638                   task->name, irq_status, task->error_mask);
2639
2640         return (task->error_mask & irq_status) ? true : false;
2641 }
2642
2643 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2644 {
2645         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2646         struct vpu_service_info *pservice = data->pservice;
2647         struct vpu_task_info *task = NULL;
2648         struct vpu_device *dev = &data->dec_dev;
2649         u32 hw_id = data->hw_info->hw_id;
2650         u32 raw_status;
2651         u32 dec_status;
2652
2653         task = &data->task_info[TASK_DEC];
2654
2655         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2656         dec_status = raw_status;
2657
2658         vpu_debug(DEBUG_TASK_INFO, "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2659                   task->reg_irq, dec_status,
2660                   task->irq_mask, task->ready_mask, task->error_mask);
2661
2662         if (dec_status & task->irq_mask) {
2663                 time_record(task, 1);
2664                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2665                           dec_status);
2666                 if ((dec_status & 0x40001) == 0x40001) {
2667                         do {
2668                                 dec_status =
2669                                         readl_relaxed(dev->regs +
2670                                                 task->reg_irq);
2671                         } while ((dec_status & 0x40001) == 0x40001);
2672                 }
2673
2674                 if (check_irq_err(task, dec_status))
2675                         atomic_add(1, &pservice->reset_request);
2676
2677                 writel_relaxed(0, dev->regs + task->reg_irq);
2678
2679                 /*
2680                  * NOTE: rkvdec need to reset after each task to avoid timeout
2681                  *       error on H.264 switch to H.265
2682                  */
2683                 if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2684                         writel(0x100000, dev->regs + task->reg_irq);
2685
2686                 /* set clock gating to save power */
2687                 writel(task->gating_mask, dev->regs + task->reg_irq);
2688
2689                 atomic_add(1, &dev->irq_count_codec);
2690                 time_diff(task);
2691         }
2692
2693         task = &data->task_info[TASK_PP];
2694         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2695                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2696
2697                 if (pp_status & task->irq_mask) {
2698                         time_record(task, 1);
2699                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2700                                   pp_status);
2701
2702                         if (check_irq_err(task, dec_status))
2703                                 atomic_add(1, &pservice->reset_request);
2704
2705                         /* clear pp IRQ */
2706                         writel_relaxed(pp_status & (~task->reg_irq),
2707                                        dev->regs + task->irq_mask);
2708                         atomic_add(1, &dev->irq_count_pp);
2709                         time_diff(task);
2710                 }
2711         }
2712
2713         pservice->irq_status = raw_status;
2714
2715         if (atomic_read(&dev->irq_count_pp) ||
2716             atomic_read(&dev->irq_count_codec))
2717                 return IRQ_WAKE_THREAD;
2718         else
2719                 return IRQ_NONE;
2720 }
2721
2722 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2723 {
2724         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2725         struct vpu_service_info *pservice = data->pservice;
2726         struct vpu_device *dev = &data->dec_dev;
2727
2728         mutex_lock(&pservice->lock);
2729         if (atomic_read(&dev->irq_count_codec)) {
2730                 atomic_sub(1, &dev->irq_count_codec);
2731                 if (pservice->reg_codec == NULL) {
2732                         vpu_err("error: dec isr with no task waiting\n");
2733                 } else {
2734                         reg_from_run_to_done(data, pservice->reg_codec);
2735                         /* avoid vpu timeout and can't recover problem */
2736                         VDPU_SOFT_RESET(data->regs);
2737                 }
2738         }
2739
2740         if (atomic_read(&dev->irq_count_pp)) {
2741                 atomic_sub(1, &dev->irq_count_pp);
2742                 if (pservice->reg_pproc == NULL)
2743                         vpu_err("error: pp isr with no task waiting\n");
2744                 else
2745                         reg_from_run_to_done(data, pservice->reg_pproc);
2746         }
2747         try_set_reg(data);
2748         mutex_unlock(&pservice->lock);
2749         return IRQ_HANDLED;
2750 }
2751
2752 static irqreturn_t vepu_irq(int irq, void *dev_id)
2753 {
2754         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2755         struct vpu_service_info *pservice = data->pservice;
2756         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2757         struct vpu_device *dev = &data->enc_dev;
2758         u32 irq_status;
2759
2760         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2761
2762         vpu_debug(DEBUG_TASK_INFO, "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2763                   task->reg_irq, irq_status,
2764                   task->irq_mask, task->ready_mask, task->error_mask);
2765
2766         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2767
2768         if (likely(irq_status & task->irq_mask)) {
2769                 time_record(task, 1);
2770
2771                 if (check_irq_err(task, irq_status))
2772                         atomic_add(1, &pservice->reset_request);
2773
2774                 /* clear enc IRQ */
2775                 writel_relaxed(irq_status & (~task->irq_mask),
2776                                dev->regs + task->reg_irq);
2777
2778                 atomic_add(1, &dev->irq_count_codec);
2779                 time_diff(task);
2780         }
2781
2782         pservice->irq_status = irq_status;
2783
2784         if (atomic_read(&dev->irq_count_codec))
2785                 return IRQ_WAKE_THREAD;
2786         else
2787                 return IRQ_NONE;
2788 }
2789
2790 static irqreturn_t vepu_isr(int irq, void *dev_id)
2791 {
2792         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2793         struct vpu_service_info *pservice = data->pservice;
2794         struct vpu_device *dev = &data->enc_dev;
2795
2796         mutex_lock(&pservice->lock);
2797         if (atomic_read(&dev->irq_count_codec)) {
2798                 atomic_sub(1, &dev->irq_count_codec);
2799                 if (NULL == pservice->reg_codec)
2800                         vpu_err("error: enc isr with no task waiting\n");
2801                 else
2802                         reg_from_run_to_done(data, pservice->reg_codec);
2803         }
2804         try_set_reg(data);
2805         mutex_unlock(&pservice->lock);
2806         return IRQ_HANDLED;
2807 }
2808
2809 static int __init vcodec_service_init(void)
2810 {
2811         int ret = platform_driver_register(&vcodec_driver);
2812
2813         if (ret) {
2814                 vpu_err("Platform device register failed (%d).\n", ret);
2815                 return ret;
2816         }
2817
2818 #ifdef CONFIG_DEBUG_FS
2819         vcodec_debugfs_init();
2820 #endif
2821
2822         return ret;
2823 }
2824
2825 static void __exit vcodec_service_exit(void)
2826 {
2827 #ifdef CONFIG_DEBUG_FS
2828         vcodec_debugfs_exit();
2829 #endif
2830
2831         platform_driver_unregister(&vcodec_driver);
2832 }
2833
2834 module_init(vcodec_service_init);
2835 module_exit(vcodec_service_exit);
2836 MODULE_LICENSE("Proprietary");
2837
2838 #ifdef CONFIG_DEBUG_FS
2839 #include <linux/seq_file.h>
2840
2841 static int vcodec_debugfs_init(void)
2842 {
2843         parent = debugfs_create_dir("vcodec", NULL);
2844         if (!parent)
2845                 return -1;
2846
2847         return 0;
2848 }
2849
2850 static void vcodec_debugfs_exit(void)
2851 {
2852         debugfs_remove(parent);
2853 }
2854
2855 static struct dentry *vcodec_debugfs_create_device_dir(
2856                 char *dirname, struct dentry *parent)
2857 {
2858         return debugfs_create_dir(dirname, parent);
2859 }
2860
2861 static int debug_vcodec_show(struct seq_file *s, void *unused)
2862 {
2863         struct vpu_subdev_data *data = s->private;
2864         struct vpu_service_info *pservice = data->pservice;
2865         unsigned int i, n;
2866         struct vpu_reg *reg, *reg_tmp;
2867         struct vpu_session *session, *session_tmp;
2868
2869         mutex_lock(&pservice->lock);
2870         vpu_service_power_on(pservice);
2871         if (data->hw_info->hw_id != HEVC_ID) {
2872                 seq_puts(s, "\nENC Registers:\n");
2873                 n = data->enc_dev.iosize >> 2;
2874
2875                 for (i = 0; i < n; i++)
2876                         seq_printf(s, "\tswreg%d = %08X\n", i,
2877                                    readl_relaxed(data->enc_dev.regs + i));
2878         }
2879
2880         seq_puts(s, "\nDEC Registers:\n");
2881
2882         n = data->dec_dev.iosize >> 2;
2883         for (i = 0; i < n; i++)
2884                 seq_printf(s, "\tswreg%d = %08X\n", i,
2885                            readl_relaxed(data->dec_dev.regs + i));
2886
2887         seq_puts(s, "\nvpu service status:\n");
2888
2889         list_for_each_entry_safe(session, session_tmp,
2890                                  &pservice->session, list_session) {
2891                 seq_printf(s, "session pid %d type %d:\n",
2892                            session->pid, session->type);
2893
2894                 list_for_each_entry_safe(reg, reg_tmp,
2895                                          &session->waiting, session_link) {
2896                         seq_printf(s, "waiting register set %p\n", reg);
2897                 }
2898                 list_for_each_entry_safe(reg, reg_tmp,
2899                                          &session->running, session_link) {
2900                         seq_printf(s, "running register set %p\n", reg);
2901                 }
2902                 list_for_each_entry_safe(reg, reg_tmp,
2903                                          &session->done, session_link) {
2904                         seq_printf(s, "done    register set %p\n", reg);
2905                 }
2906         }
2907
2908         seq_printf(s, "\npower counter: on %d off %d\n",
2909                    atomic_read(&pservice->power_on_cnt),
2910                    atomic_read(&pservice->power_off_cnt));
2911
2912         mutex_unlock(&pservice->lock);
2913         vpu_service_power_off(pservice);
2914
2915         return 0;
2916 }
2917
2918 static int debug_vcodec_open(struct inode *inode, struct file *file)
2919 {
2920         return single_open(file, debug_vcodec_show, inode->i_private);
2921 }
2922
2923 #endif
2924