drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming, chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *         Jung Zhao, jung.zhao@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/clk.h>
  21 #include <linux/compat.h>
  22 #include <linux/delay.h>
  23 #include <linux/init.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/mm.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/reset.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/wakelock.h>
  33 #include <linux/cdev.h>
  34 #include <linux/of.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/regmap.h>
  38 #include <linux/mfd/syscon.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/iopoll.h>
  43
  44 #include <linux/rockchip/cru.h>
  45 #include <linux/rockchip/pmu.h>
  46 #include <linux/rockchip/grf.h>
  47
  48 #include <linux/dma-buf.h>
  49 #include <linux/rockchip-iovmm.h>
  50
  51 #include "vcodec_hw_info.h"
  52 #include "vcodec_hw_vpu.h"
  53 #include "vcodec_hw_rkv.h"
  54 #include "vcodec_hw_vpu2.h"
  55
  56 #include "vcodec_service.h"
  57
  58 #include "vcodec_iommu_ops.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 #define MHZ                                     (1000 * 1000)
  90 #define SIZE_REG(reg)                           ((reg) * 4)
  91
  92 #define VCODEC_CLOCK_ENABLE     1
  93 #define EXTRA_INFO_MAGIC        0x4C4A46
  94
  95 static int debug;
  96 module_param(debug, int, S_IRUGO | S_IWUSR);
  97 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  98 /*
  99  * hardware information organization
 100  *
 101  * In order to support multiple hardware with different version the hardware
 102  * information is organized as follow:
 103  *
 104  * 1. First, index hardware by register size / position.
 105  *    These information is fix for each hardware and do not relate to runtime
 106  *    work flow. It only related to resource allocation.
 107  *    Descriptor: struct vpu_hw_info
 108  *
 109  * 2. Then, index hardware by runtime configuration
 110  *    These information is related to runtime setting behave including enable
 111  *    register, irq register and other key control flag
 112  *    Descriptor: struct vpu_task_info
 113  *
 114  * 3. Final, on iommu case the fd translation is required
 115  *    Descriptor: struct vpu_trans_info
 116  */
 117
 118 enum VPU_FREQ {
 119         VPU_FREQ_200M,
 120         VPU_FREQ_266M,
 121         VPU_FREQ_300M,
 122         VPU_FREQ_400M,
 123         VPU_FREQ_500M,
 124         VPU_FREQ_600M,
 125         VPU_FREQ_DEFAULT,
 126         VPU_FREQ_BUT,
 127 };
 128
 129 struct extra_info_elem {
 130         u32 index;
 131         u32 offset;
 132 };
 133
 134
 135 struct extra_info_for_iommu {
 136         u32 magic;
 137         u32 cnt;
 138         struct extra_info_elem elem[20];
 139 };
 140
 141 static const struct vcodec_info vcodec_info_set[] = {
 142         {
 143                 .hw_id          = VPU_ID_8270,
 144                 .hw_info        = &hw_vpu_8270,
 145                 .task_info      = task_vpu,
 146                 .trans_info     = trans_vpu,
 147         },
 148         {
 149                 .hw_id          = VPU_ID_4831,
 150                 .hw_info        = &hw_vpu_4831,
 151                 .task_info      = task_vpu,
 152                 .trans_info     = trans_vpu,
 153         },
 154         {
 155                 .hw_id          = VPU_DEC_ID_9190,
 156                 .hw_info        = &hw_vpu_9190,
 157                 .task_info      = task_vpu,
 158                 .trans_info     = trans_vpu,
 159         },
 160         {
 161                 .hw_id          = HEVC_ID,
 162                 .hw_info        = &hw_rkhevc,
 163                 .task_info      = task_rkv,
 164                 .trans_info     = trans_rkv,
 165         },
 166         {
 167                 .hw_id          = RKV_DEC_ID,
 168                 .hw_info        = &hw_rkvdec,
 169                 .task_info      = task_rkv,
 170                 .trans_info     = trans_rkv,
 171         },
 172         {
 173                 .hw_id          = VPU2_ID,
 174                 .hw_info        = &hw_vpu2,
 175                 .task_info      = task_vpu2,
 176                 .trans_info     = trans_vpu2,
 177         },
 178 };
 179
 180 /* Both VPU1 and VPU2 */
 181 static const struct vcodec_device_info vpu_device_info = {
 182         .device_type = VCODEC_DEVICE_TYPE_VPUX,
 183         .name = "vpu-service",
 184 };
 185
 186 static const struct vcodec_device_info vpu_combo_device_info = {
 187         .device_type = VCODEC_DEVICE_TYPE_VPUC,
 188         .name = "vpu-combo",
 189 };
 190
 191 static const struct vcodec_device_info hevc_device_info = {
 192         .device_type = VCODEC_DEVICE_TYPE_HEVC,
 193         .name = "hevc-service",
 194 };
 195
 196 static const struct vcodec_device_info rkvd_device_info = {
 197         .device_type = VCODEC_DEVICE_TYPE_RKVD,
 198         .name = "rkvdec",
 199 };
 200
 201 #define DEBUG
 202 #ifdef DEBUG
 203 #define vpu_debug_func(type, fmt, args...)                      \
 204         do {                                                    \
 205                 if (unlikely(debug & type)) {                   \
 206                         pr_info("%s:%d: " fmt,                  \
 207                                  __func__, __LINE__, ##args);   \
 208                 }                                               \
 209         } while (0)
 210 #define vpu_debug(type, fmt, args...)                           \
 211         do {                                                    \
 212                 if (unlikely(debug & type)) {                   \
 213                         pr_info(fmt, ##args);                   \
 214                 }                                               \
 215         } while (0)
 216 #else
 217 #define vpu_debug_func(level, fmt, args...)
 218 #define vpu_debug(level, fmt, args...)
 219 #endif
 220
 221 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 222 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 223
 224 #define vpu_err(fmt, args...)                           \
 225                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 226
 227 enum VPU_DEC_FMT {
 228         VPU_DEC_FMT_H264,
 229         VPU_DEC_FMT_MPEG4,
 230         VPU_DEC_FMT_H263,
 231         VPU_DEC_FMT_JPEG,
 232         VPU_DEC_FMT_VC1,
 233         VPU_DEC_FMT_MPEG2,
 234         VPU_DEC_FMT_MPEG1,
 235         VPU_DEC_FMT_VP6,
 236         VPU_DEC_FMT_RESERV0,
 237         VPU_DEC_FMT_VP7,
 238         VPU_DEC_FMT_VP8,
 239         VPU_DEC_FMT_AVS,
 240         VPU_DEC_FMT_RES
 241 };
 242
 243 /**
 244  * struct for process session which connect to vpu
 245  *
 246  * @author ChenHengming (2011-5-3)
 247  */
 248 struct vpu_session {
 249         enum VPU_CLIENT_TYPE type;
 250         /* a linked list of data so we can access them for debugging */
 251         struct list_head list_session;
 252         /* a linked list of register data waiting for process */
 253         struct list_head waiting;
 254         /* a linked list of register data in processing */
 255         struct list_head running;
 256         /* a linked list of register data processed */
 257         struct list_head done;
 258         wait_queue_head_t wait;
 259         pid_t pid;
 260         atomic_t task_running;
 261 };
 262
 263 /**
 264  * struct for process register set
 265  *
 266  * @author ChenHengming (2011-5-4)
 267  */
 268 struct vpu_reg {
 269         enum VPU_CLIENT_TYPE type;
 270         enum VPU_FREQ freq;
 271         struct vpu_session *session;
 272         struct vpu_subdev_data *data;
 273         struct vpu_task_info *task;
 274         const struct vpu_trans_info *trans;
 275
 276         /* link to vpu service session */
 277         struct list_head session_link;
 278         /* link to register set list */
 279         struct list_head status_link;
 280
 281         unsigned long size;
 282         struct list_head mem_region_list;
 283         u32 dec_base;
 284         u32 *reg;
 285 };
 286
 287 struct vpu_device {
 288         atomic_t irq_count_codec;
 289         atomic_t irq_count_pp;
 290         unsigned int iosize;
 291         u32 *regs;
 292 };
 293
 294 enum vcodec_device_id {
 295         VCODEC_DEVICE_ID_VPU,
 296         VCODEC_DEVICE_ID_HEVC,
 297         VCODEC_DEVICE_ID_COMBO,
 298         VCODEC_DEVICE_ID_RKVDEC,
 299         VCODEC_DEVICE_ID_BUTT
 300 };
 301
 302 enum VCODEC_RUNNING_MODE {
 303         VCODEC_RUNNING_MODE_NONE = -1,
 304         VCODEC_RUNNING_MODE_VPU,
 305         VCODEC_RUNNING_MODE_HEVC,
 306         VCODEC_RUNNING_MODE_RKVDEC
 307 };
 308
 309 struct vcodec_mem_region {
 310         struct list_head srv_lnk;
 311         struct list_head reg_lnk;
 312         struct list_head session_lnk;
 313         unsigned long iova;     /* virtual address for iommu */
 314         unsigned long len;
 315         u32 reg_idx;
 316         int hdl;
 317 };
 318
 319 enum vpu_ctx_state {
 320         MMU_ACTIVATED   = BIT(0)
 321 };
 322
 323 struct vpu_subdev_data {
 324         struct cdev cdev;
 325         dev_t dev_t;
 326         struct class *cls;
 327         struct device *child_dev;
 328
 329         int irq_enc;
 330         int irq_dec;
 331         struct vpu_service_info *pservice;
 332
 333         u32 *regs;
 334         enum VCODEC_RUNNING_MODE mode;
 335         struct list_head lnk_service;
 336
 337         struct device *dev;
 338
 339         struct vpu_device enc_dev;
 340         struct vpu_device dec_dev;
 341
 342         enum VPU_HW_ID hw_id;
 343         struct vpu_hw_info *hw_info;
 344         struct vpu_task_info *task_info;
 345         const struct vpu_trans_info *trans_info;
 346
 347         u32 reg_size;
 348         unsigned long state;
 349
 350 #ifdef CONFIG_DEBUG_FS
 351         struct dentry *debugfs_dir;
 352         struct dentry *debugfs_file_regs;
 353 #endif
 354
 355         struct device *mmu_dev;
 356         struct vcodec_iommu_info *iommu_info;
 357         struct work_struct set_work;
 358 };
 359
 360 struct vpu_service_info {
 361         struct wake_lock wake_lock;
 362         struct delayed_work power_off_work;
 363         struct wake_lock set_wake_lock;
 364         struct workqueue_struct *set_workq;
 365         ktime_t last; /* record previous power-on time */
 366         /* vpu service structure global lock */
 367         struct mutex lock;
 368         /* link to link_reg in struct vpu_reg */
 369         struct list_head waiting;
 370         /* link to link_reg in struct vpu_reg */
 371         struct list_head running;
 372         /* link to link_reg in struct vpu_reg */
 373         struct list_head done;
 374         /* link to list_session in struct vpu_session */
 375         struct list_head session;
 376         atomic_t total_running;
 377         atomic_t enabled;
 378         atomic_t power_on_cnt;
 379         atomic_t power_off_cnt;
 380         atomic_t service_on;
 381         struct mutex shutdown_lock;
 382         struct vpu_reg *reg_codec;
 383         struct vpu_reg *reg_pproc;
 384         struct vpu_reg *reg_resev;
 385         struct vpu_dec_config dec_config;
 386         struct vpu_enc_config enc_config;
 387
 388         bool auto_freq;
 389         bool bug_dec_addr;
 390         atomic_t freq_status;
 391
 392         struct clk *aclk_vcodec;
 393         struct clk *hclk_vcodec;
 394         struct clk *clk_core;
 395         struct clk *clk_cabac;
 396         struct clk *pd_video;
 397
 398 #ifdef CONFIG_RESET_CONTROLLER
 399         struct reset_control *rst_a;
 400         struct reset_control *rst_h;
 401         struct reset_control *rst_v;
 402 #endif
 403         struct device *dev;
 404
 405         u32 irq_status;
 406         atomic_t reset_request;
 407         struct list_head mem_region_list;
 408
 409         enum vcodec_device_id dev_id;
 410
 411         enum VCODEC_RUNNING_MODE curr_mode;
 412         u32 prev_mode;
 413
 414         struct delayed_work simulate_work;
 415
 416         u32 mode_bit;
 417         u32 mode_ctrl;
 418         u32 *reg_base;
 419         u32 ioaddr;
 420         struct regmap *grf;
 421         u32 *grf_base;
 422
 423         char *name;
 424
 425         u32 subcnt;
 426         struct list_head subdev_list;
 427
 428         u32 alloc_type;
 429 };
 430
 431 struct vpu_request {
 432         u32 *req;
 433         u32 size;
 434 };
 435
 436 #ifdef CONFIG_COMPAT
 437 struct compat_vpu_request {
 438         compat_uptr_t req;
 439         u32 size;
 440 };
 441 #endif
 442
 443 #define VDPU_SOFT_RESET_REG     101
 444 #define VDPU_CLEAN_CACHE_REG    516
 445 #define VEPU_CLEAN_CACHE_REG    772
 446 #define HEVC_CLEAN_CACHE_REG    260
 447
 448 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 449
 450 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 451 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 452 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 453 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 454
 455 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 456 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 457
 458 static void *vcodec_get_drv_data(struct platform_device *pdev);
 459
 460 static void vpu_service_power_on(struct vpu_subdev_data *data,
 461                                  struct vpu_service_info *pservice);
 462
 463 static void time_record(struct vpu_task_info *task, int is_end)
 464 {
 465         if (unlikely(debug & DEBUG_TIMING) && task)
 466                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 467 }
 468
 469 static void time_diff(struct vpu_task_info *task)
 470 {
 471         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 472                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 473                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 474 }
 475
 476 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 477 {
 478         int bits;
 479         u32 raw = 0;
 480         struct vpu_service_info *pservice = data->pservice;
 481         struct vpu_subdev_data *subdata, *n;
 482
 483         if (pservice->subcnt < 2) {
 484                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 485                         set_bit(MMU_ACTIVATED, &data->state);
 486
 487                         if (atomic_read(&pservice->enabled)) {
 488                                 if (vcodec_iommu_attach(data->iommu_info))
 489                                         dev_err(data->dev,
 490                                                 "vcodec service attach failed\n"
 491                                                 );
 492                                 else
 493                                         BUG_ON(
 494                                                !atomic_read(&pservice->enabled)
 495                                                );
 496                         }
 497                 }
 498                 return;
 499         }
 500
 501         if (pservice->curr_mode == data->mode)
 502                 return;
 503
 504         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 505         list_for_each_entry_safe(subdata, n,
 506                                  &pservice->subdev_list, lnk_service) {
 507                 if (data != subdata && subdata->mmu_dev &&
 508                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 509                         clear_bit(MMU_ACTIVATED, &subdata->state);
 510                         vcodec_iommu_detach(subdata->iommu_info);
 511                 }
 512         }
 513         bits = 1 << pservice->mode_bit;
 514 #ifdef CONFIG_MFD_SYSCON
 515         if (pservice->grf) {
 516                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 517
 518                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 519                         regmap_write(pservice->grf, pservice->mode_ctrl,
 520                                      raw | bits | (bits << 16));
 521                 else
 522                         regmap_write(pservice->grf, pservice->mode_ctrl,
 523                                      (raw & (~bits)) | (bits << 16));
 524         } else if (pservice->grf_base) {
 525                 u32 *grf_base = pservice->grf_base;
 526
 527                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 528                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 529                         writel_relaxed(raw | bits | (bits << 16),
 530                                        grf_base + pservice->mode_ctrl / 4);
 531                 else
 532                         writel_relaxed((raw & (~bits)) | (bits << 16),
 533                                        grf_base + pservice->mode_ctrl / 4);
 534         } else {
 535                 vpu_err("no grf resource define, switch decoder failed\n");
 536                 return;
 537         }
 538 #else
 539         if (pservice->grf_base) {
 540                 u32 *grf_base = pservice->grf_base;
 541
 542                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 543                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 544                         writel_relaxed(raw | bits | (bits << 16),
 545                                        grf_base + pservice->mode_ctrl / 4);
 546                 else
 547                         writel_relaxed((raw & (~bits)) | (bits << 16),
 548                                        grf_base + pservice->mode_ctrl / 4);
 549         } else {
 550                 vpu_err("no grf resource define, switch decoder failed\n");
 551                 return;
 552         }
 553 #endif
 554         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 555                 set_bit(MMU_ACTIVATED, &data->state);
 556                 if (atomic_read(&pservice->enabled))
 557                         vcodec_iommu_attach(data->iommu_info);
 558                 else
 559                         /* FIXME BUG_ON should not be used in mass produce */
 560                         BUG_ON(!atomic_read(&pservice->enabled));
 561         }
 562
 563         pservice->prev_mode = pservice->curr_mode;
 564         pservice->curr_mode = data->mode;
 565 }
 566
 567 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 568 {
 569         /*
 570          * In case of VPU Combo, it require HW switch its running mode
 571          * before the other HW component start work. set current HW running
 572          * mode to none, can ensure HW switch to its reqired mode properly.
 573          */
 574         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 575 }
 576
 577 static int vpu_get_clk(struct vpu_service_info *pservice)
 578 {
 579 #if VCODEC_CLOCK_ENABLE
 580         struct device *dev = pservice->dev;
 581
 582         switch (pservice->dev_id) {
 583         case VCODEC_DEVICE_ID_HEVC:
 584                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 585                 if (IS_ERR(pservice->pd_video)) {
 586                         pservice->pd_video = NULL;
 587                         dev_info(dev, "failed on clk_get pd_hevc\n");
 588                 }
 589         case VCODEC_DEVICE_ID_COMBO:
 590         case VCODEC_DEVICE_ID_RKVDEC:
 591                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 592                 if (IS_ERR(pservice->clk_cabac)) {
 593                         dev_err(dev, "failed on clk_get clk_cabac\n");
 594                         pservice->clk_cabac = NULL;
 595                 }
 596                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 597                 if (IS_ERR(pservice->clk_core)) {
 598                         dev_err(dev, "failed on clk_get clk_core\n");
 599                         pservice->clk_core = NULL;
 600                         return -1;
 601                 }
 602         case VCODEC_DEVICE_ID_VPU:
 603                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 604                 if (IS_ERR(pservice->aclk_vcodec)) {
 605                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 606                         pservice->aclk_vcodec = NULL;
 607                         return -1;
 608                 }
 609
 610                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 611                 if (IS_ERR(pservice->hclk_vcodec)) {
 612                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 613                         pservice->hclk_vcodec = NULL;
 614                         return -1;
 615                 }
 616                 if (pservice->pd_video == NULL) {
 617                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 618                         if (IS_ERR(pservice->pd_video)) {
 619                                 pservice->pd_video = NULL;
 620                                 dev_info(dev, "do not have pd_video\n");
 621                         }
 622                 }
 623                 break;
 624         default:
 625                 break;
 626         }
 627
 628         return 0;
 629 #else
 630         return 0;
 631 #endif
 632 }
 633
 634 static void _vpu_reset(struct vpu_subdev_data *data)
 635 {
 636         struct vpu_service_info *pservice = data->pservice;
 637         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 638
 639         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 640                 type = IDLE_REQ_HEVC;
 641
 642         dev_info(pservice->dev, "resetting...\n");
 643         WARN_ON(pservice->reg_codec != NULL);
 644         WARN_ON(pservice->reg_pproc != NULL);
 645         WARN_ON(pservice->reg_resev != NULL);
 646         pservice->reg_codec = NULL;
 647         pservice->reg_pproc = NULL;
 648         pservice->reg_resev = NULL;
 649
 650 #ifdef CONFIG_RESET_CONTROLLER
 651         dev_info(pservice->dev, "for 3288/3368...");
 652         if (of_machine_is_compatible("rockchip,rk3288"))
 653                 rockchip_pmu_idle_request(pservice->dev, true);
 654         if (pservice->rst_a && pservice->rst_h) {
 655                 dev_info(pservice->dev, "vpu reset in\n");
 656
 657                 if (pservice->rst_v)
 658                         reset_control_assert(pservice->rst_v);
 659                 reset_control_assert(pservice->rst_a);
 660                 reset_control_assert(pservice->rst_h);
 661                 udelay(5);
 662
 663                 reset_control_deassert(pservice->rst_h);
 664                 reset_control_deassert(pservice->rst_a);
 665                 if (pservice->rst_v)
 666                         reset_control_deassert(pservice->rst_v);
 667         } else if (pservice->rst_v) {
 668                 dev_info(pservice->dev, "hevc reset in\n");
 669                 reset_control_assert(pservice->rst_v);
 670                 udelay(5);
 671
 672                 reset_control_deassert(pservice->rst_v);
 673         }
 674         if (of_machine_is_compatible("rockchip,rk3288"))
 675                 rockchip_pmu_idle_request(pservice->dev, false);
 676 #endif
 677 }
 678
 679 static void vpu_reset(struct vpu_subdev_data *data)
 680 {
 681         struct vpu_service_info *pservice = data->pservice;
 682
 683         _vpu_reset(data);
 684         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 685                 if (atomic_read(&pservice->enabled)) {
 686                         /* Need to reset iommu */
 687                         vcodec_iommu_detach(data->iommu_info);
 688                 } else {
 689                         /* FIXME BUG_ON should not be used in mass produce */
 690                         BUG_ON(!atomic_read(&pservice->enabled));
 691                 }
 692         }
 693
 694         atomic_set(&pservice->reset_request, 0);
 695         dev_info(pservice->dev, "reset done\n");
 696 }
 697
 698 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 699 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 700                                       struct vpu_session *session)
 701 {
 702         struct vpu_reg *reg, *n;
 703
 704         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 705                 reg_deinit(data, reg);
 706         }
 707         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 708                 reg_deinit(data, reg);
 709         }
 710         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 711                 reg_deinit(data, reg);
 712         }
 713 }
 714
 715 static void vpu_service_clear(struct vpu_subdev_data *data)
 716 {
 717         struct vpu_reg *reg, *n;
 718         struct vpu_session *session, *s;
 719         struct vpu_service_info *pservice = data->pservice;
 720
 721         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 722                 reg_deinit(reg->data, reg);
 723         }
 724
 725         /* wake up session wait event to prevent the timeout hw reset
 726          * during reboot procedure.
 727          */
 728         list_for_each_entry_safe(session, s,
 729                                  &pservice->session, list_session)
 730                 wake_up(&session->wait);
 731 }
 732
 733 static void vpu_service_dump(struct vpu_service_info *pservice)
 734 {
 735 }
 736
 737
 738 static void vpu_service_power_off(struct vpu_service_info *pservice)
 739 {
 740         int total_running;
 741         struct vpu_subdev_data *data = NULL, *n;
 742         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 743
 744         if (!ret)
 745                 return;
 746
 747         total_running = atomic_read(&pservice->total_running);
 748         if (total_running) {
 749                 pr_alert("alert: power off when %d task running!!\n",
 750                          total_running);
 751                 mdelay(50);
 752                 pr_alert("alert: delay 50 ms for running task\n");
 753                 vpu_service_dump(pservice);
 754         }
 755
 756         dev_dbg(pservice->dev, "power off...\n");
 757
 758         udelay(5);
 759
 760         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 761                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 762                         clear_bit(MMU_ACTIVATED, &data->state);
 763                         vcodec_iommu_detach(data->iommu_info);
 764                 }
 765         }
 766         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 767         pm_runtime_put(pservice->dev);
 768 #if VCODEC_CLOCK_ENABLE
 769                 if (pservice->pd_video)
 770                         clk_disable_unprepare(pservice->pd_video);
 771                 if (pservice->hclk_vcodec)
 772                         clk_disable_unprepare(pservice->hclk_vcodec);
 773                 if (pservice->aclk_vcodec)
 774                         clk_disable_unprepare(pservice->aclk_vcodec);
 775                 if (pservice->clk_core)
 776                         clk_disable_unprepare(pservice->clk_core);
 777                 if (pservice->clk_cabac)
 778                         clk_disable_unprepare(pservice->clk_cabac);
 779 #endif
 780
 781         atomic_add(1, &pservice->power_off_cnt);
 782         wake_unlock(&pservice->wake_lock);
 783         dev_dbg(pservice->dev, "power off done\n");
 784 }
 785
 786 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 787 {
 788         queue_delayed_work(system_wq, &pservice->power_off_work,
 789                            VPU_POWER_OFF_DELAY);
 790 }
 791
 792 static void vpu_power_off_work(struct work_struct *work_s)
 793 {
 794         struct delayed_work *dlwork = container_of(work_s,
 795                         struct delayed_work, work);
 796         struct vpu_service_info *pservice = container_of(dlwork,
 797                         struct vpu_service_info, power_off_work);
 798
 799         if (mutex_trylock(&pservice->lock)) {
 800                 vpu_service_power_off(pservice);
 801                 mutex_unlock(&pservice->lock);
 802         } else {
 803                 /* Come back later if the device is busy... */
 804                 vpu_queue_power_off_work(pservice);
 805         }
 806 }
 807
 808 static void vpu_service_power_on(struct vpu_subdev_data *data,
 809                                  struct vpu_service_info *pservice)
 810 {
 811         int ret;
 812         ktime_t now = ktime_get();
 813
 814         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC ||
 815             atomic_read(&pservice->power_on_cnt)) {
 816                 /* NSEC_PER_SEC */
 817                 cancel_delayed_work_sync(&pservice->power_off_work);
 818                 vpu_queue_power_off_work(pservice);
 819                 pservice->last = now;
 820         }
 821         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 822         if (!ret)
 823                 return;
 824
 825         dev_dbg(pservice->dev, "power on\n");
 826
 827 #define BIT_VCODEC_CLK_SEL      (1<<10)
 828         if (of_machine_is_compatible("rockchip,rk3126"))
 829                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 830                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 831                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 832
 833 #if VCODEC_CLOCK_ENABLE
 834         if (pservice->aclk_vcodec)
 835                 clk_prepare_enable(pservice->aclk_vcodec);
 836         if (pservice->hclk_vcodec)
 837                 clk_prepare_enable(pservice->hclk_vcodec);
 838         if (pservice->clk_core)
 839                 clk_prepare_enable(pservice->clk_core);
 840         if (pservice->clk_cabac)
 841                 clk_prepare_enable(pservice->clk_cabac);
 842         if (pservice->pd_video)
 843                 clk_prepare_enable(pservice->pd_video);
 844 #endif
 845         pm_runtime_get_sync(pservice->dev);
 846
 847         udelay(5);
 848         atomic_add(1, &pservice->power_on_cnt);
 849         wake_lock(&pservice->wake_lock);
 850 }
 851
 852 static inline bool reg_check_interlace(struct vpu_reg *reg)
 853 {
 854         u32 type = (reg->reg[3] & (1 << 23));
 855
 856         return (type > 0);
 857 }
 858
 859 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 860 {
 861         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 862
 863         return type;
 864 }
 865
 866 static inline int reg_probe_width(struct vpu_reg *reg)
 867 {
 868         int width_in_mb = reg->reg[4] >> 23;
 869
 870         return width_in_mb * 16;
 871 }
 872
 873 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 874 {
 875         int y_virstride = reg->reg[8];
 876
 877         return y_virstride;
 878 }
 879
 880 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 881                 struct vpu_session *session,
 882                 struct vpu_reg *reg,
 883                 int fd)
 884 {
 885         int hdl;
 886         int ret = 0;
 887         struct vcodec_mem_region *mem_region;
 888
 889         hdl = vcodec_iommu_import(data->iommu_info, session, fd);
 890         if (hdl < 0)
 891                 return hdl;
 892
 893         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 894         if (mem_region == NULL) {
 895                 vpu_err("allocate memory for iommu memory region failed\n");
 896                 vcodec_iommu_free(data->iommu_info, session, hdl);
 897                 return -ENOMEM;
 898         }
 899
 900         mem_region->hdl = hdl;
 901         ret = vcodec_iommu_map_iommu(data->iommu_info, session, mem_region->hdl,
 902                                      &mem_region->iova, &mem_region->len);
 903         if (ret < 0) {
 904                 vpu_err("fd %d ion map iommu failed\n", fd);
 905                 kfree(mem_region);
 906                 vcodec_iommu_free(data->iommu_info, session, hdl);
 907
 908                 return -EFAULT;
 909         }
 910         INIT_LIST_HEAD(&mem_region->reg_lnk);
 911         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 912         return mem_region->iova;
 913 }
 914
 915 /*
 916  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 917  * it by pps id in video stream data.
 918  *
 919  * So we need to translate the address in iommu case. The address data is also
 920  * 10bit fd + 22bit offset mode.
 921  * Because userspace decoder do not give the pps id in the register file sets
 922  * kernel driver need to translate each scaling list address in pps buffer which
 923  * means 256 pps for H.264, 64 pps for H.265.
 924  *
 925  * In order to optimize the performance kernel driver ask userspace decoder to
 926  * set all scaling list address in pps buffer to the same one which will be used
 927  * on current decoding task. Then kernel driver can only translate the first
 928  * address then copy it all pps buffer.
 929  */
 930 static int fill_scaling_list_addr_in_pps(
 931                 struct vpu_subdev_data *data,
 932                 struct vpu_reg *reg,
 933                 char *pps,
 934                 int pps_info_count,
 935                 int pps_info_size,
 936                 int scaling_list_addr_offset)
 937 {
 938         int base = scaling_list_addr_offset;
 939         int scaling_fd = 0;
 940         u32 scaling_offset;
 941
 942         scaling_offset  = (u32)pps[base + 0];
 943         scaling_offset += (u32)pps[base + 1] << 8;
 944         scaling_offset += (u32)pps[base + 2] << 16;
 945         scaling_offset += (u32)pps[base + 3] << 24;
 946
 947         scaling_fd = scaling_offset & 0x3ff;
 948         scaling_offset = scaling_offset >> 10;
 949
 950         if (scaling_fd > 0) {
 951                 int i = 0;
 952                 u32 tmp = vcodec_fd_to_iova(data, reg->session, reg,
 953                                             scaling_fd);
 954
 955                 if (IS_ERR_VALUE(tmp))
 956                         return -1;
 957                 tmp += scaling_offset;
 958
 959                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 960                         pps[base + 0] = (tmp >>  0) & 0xff;
 961                         pps[base + 1] = (tmp >>  8) & 0xff;
 962                         pps[base + 2] = (tmp >> 16) & 0xff;
 963                         pps[base + 3] = (tmp >> 24) & 0xff;
 964                 }
 965         }
 966
 967         return 0;
 968 }
 969
 970 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data,
 971                                 struct vpu_session *session,
 972                                 const u8 *tbl,
 973                                 int size, struct vpu_reg *reg,
 974                                 struct extra_info_for_iommu *ext_inf)
 975 {
 976         struct vpu_service_info *pservice = data->pservice;
 977         struct vpu_task_info *task = reg->task;
 978         enum FORMAT_TYPE type;
 979         int hdl;
 980         int ret = 0;
 981         struct vcodec_mem_region *mem_region;
 982         int i;
 983         int offset = 0;
 984
 985         if (tbl == NULL || size <= 0) {
 986                 dev_err(pservice->dev, "input arguments invalidate\n");
 987                 return -EINVAL;
 988         }
 989
 990         if (task->get_fmt)
 991                 type = task->get_fmt(reg->reg);
 992         else {
 993                 dev_err(pservice->dev, "invalid task with NULL get_fmt\n");
 994                 return -EINVAL;
 995         }
 996
 997         for (i = 0; i < size; i++) {
 998                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
 999
1000                 /* if userspace do not set the fd at this register, skip */
1001                 if (usr_fd == 0)
1002                         continue;
1003
1004                 /*
1005                  * for avoiding cache sync issue, we need to map/unmap
1006                  * input buffer every time. FIX ME, if it is unnecessary
1007                  */
1008                 if (task->reg_rlc == tbl[i])
1009                         vcodec_iommu_free_fd(data->iommu_info, session, usr_fd);
1010                 /*
1011                  * special offset scale case
1012                  *
1013                  * This translation is for fd + offset translation.
1014                  * One register has 32bits. We need to transfer both buffer file
1015                  * handle and the start address offset so we packet file handle
1016                  * and offset together using below format.
1017                  *
1018                  *  0~9  bit for buffer file handle range 0 ~ 1023
1019                  * 10~31 bit for offset range 0 ~ 4M
1020                  *
1021                  * But on 4K case the offset can be larger the 4M
1022                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1023                  * But MPEG4 will use the same register for colmv and it do not
1024                  * need scale.
1025                  *
1026                  * RKVdec do not have this issue.
1027                  */
1028                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1029                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1030                         offset = reg->reg[tbl[i]] >> 10 << 4;
1031                 else
1032                         offset = reg->reg[tbl[i]] >> 10;
1033
1034                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d i %d\n",
1035                           tbl[i], usr_fd, offset, i);
1036
1037                 hdl = vcodec_iommu_import(data->iommu_info, session, usr_fd);
1038
1039                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1040                         int pps_info_offset;
1041                         int pps_info_count;
1042                         int pps_info_size;
1043                         int scaling_list_addr_offset;
1044
1045                         switch (type) {
1046                         case FMT_H264D: {
1047                                 pps_info_offset = offset;
1048                                 pps_info_count = 256;
1049                                 pps_info_size = 32;
1050                                 scaling_list_addr_offset = 23;
1051                         } break;
1052                         case FMT_H265D: {
1053                                 pps_info_offset = 0;
1054                                 pps_info_count = 64;
1055                                 pps_info_size = 80;
1056                                 scaling_list_addr_offset = 74;
1057                         } break;
1058                         default: {
1059                                 pps_info_offset = 0;
1060                                 pps_info_count = 0;
1061                                 pps_info_size = 0;
1062                                 scaling_list_addr_offset = 0;
1063                         } break;
1064                         }
1065
1066                         vpu_debug(DEBUG_PPS_FILL,
1067                                   "scaling list filling parameter:\n");
1068                         vpu_debug(DEBUG_PPS_FILL,
1069                                   "pps_info_offset %d\n", pps_info_offset);
1070                         vpu_debug(DEBUG_PPS_FILL,
1071                                   "pps_info_count  %d\n", pps_info_count);
1072                         vpu_debug(DEBUG_PPS_FILL,
1073                                   "pps_info_size   %d\n", pps_info_size);
1074                         vpu_debug(DEBUG_PPS_FILL,
1075                                   "scaling_list_addr_offset %d\n",
1076                                   scaling_list_addr_offset);
1077
1078                         if (pps_info_count) {
1079                                 u8 *pps;
1080
1081                                 pps = vcodec_iommu_map_kernel
1082                                         (data->iommu_info, session, hdl);
1083
1084                                 vpu_debug(DEBUG_PPS_FILL,
1085                                           "scaling list setting pps %p\n", pps);
1086                                 pps += pps_info_offset;
1087
1088                                 fill_scaling_list_addr_in_pps
1089                                         (data, reg, pps, pps_info_count,
1090                                          pps_info_size,
1091                                          scaling_list_addr_offset);
1092
1093                                 vcodec_iommu_unmap_kernel
1094                                         (data->iommu_info, session, hdl);
1095                         }
1096                 }
1097
1098                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1099
1100                 if (!mem_region) {
1101                         vcodec_iommu_free(data->iommu_info, session, hdl);
1102                         return -ENOMEM;
1103                 }
1104
1105                 mem_region->hdl = hdl;
1106                 mem_region->reg_idx = tbl[i];
1107
1108                 ret = vcodec_iommu_map_iommu(data->iommu_info, session,
1109                                              mem_region->hdl, &mem_region->iova,
1110                                              &mem_region->len);
1111                 if (ret < 0) {
1112                         dev_err(pservice->dev,
1113                                 "reg %d fd %d ion map iommu failed\n",
1114                                 tbl[i], usr_fd);
1115                         kfree(mem_region);
1116                         vcodec_iommu_free(data->iommu_info, session, hdl);
1117                         return ret;
1118                 }
1119
1120                 /*
1121                  * special for vpu dec num 12: record decoded length
1122                  * hacking for decoded length
1123                  * NOTE: not a perfect fix, the fd is not recorded
1124                  */
1125                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1126                         reg->dec_base = mem_region->iova + offset;
1127                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1128                                   reg->dec_base);
1129                 }
1130
1131                 reg->reg[tbl[i]] = mem_region->iova + offset;
1132                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1133                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1134         }
1135
1136         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1137                 for (i = 0; i < ext_inf->cnt; i++) {
1138                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1139                                   ext_inf->elem[i].index,
1140                                   ext_inf->elem[i].offset);
1141                         reg->reg[ext_inf->elem[i].index] +=
1142                                 ext_inf->elem[i].offset;
1143                 }
1144         }
1145
1146         return 0;
1147 }
1148
1149 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1150                                         struct vpu_session *session,
1151                                         struct vpu_reg *reg,
1152                                         struct extra_info_for_iommu *ext_inf)
1153 {
1154         struct vpu_service_info *pservice = data->pservice;
1155         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1156
1157         if (type < FMT_TYPE_BUTT) {
1158                 const struct vpu_trans_info *info = &reg->trans[type];
1159                 const u8 *tbl = info->table;
1160                 int size = info->count;
1161
1162                 return vcodec_bufid_to_iova(data, session, tbl, size, reg,
1163                                             ext_inf);
1164         }
1165
1166         dev_err(pservice->dev, "found invalid format type!\n");
1167         return -EINVAL;
1168 }
1169
1170 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1171 {
1172
1173         if (!of_machine_is_compatible("rockchip,rk2928g")) {
1174                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1175                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1176                                 if (reg_probe_width(reg) > 3200) {
1177                                         /*raise frequency for 4k avc.*/
1178                                         reg->freq = VPU_FREQ_600M;
1179                                 }
1180                         } else {
1181                                 if (reg_check_interlace(reg))
1182                                         reg->freq = VPU_FREQ_400M;
1183                         }
1184                 }
1185                 if (data->hw_id == HEVC_ID) {
1186                         if (reg_probe_hevc_y_stride(reg) > 60000)
1187                                 reg->freq = VPU_FREQ_400M;
1188                 }
1189                 if (reg->type == VPU_PP)
1190                         reg->freq = VPU_FREQ_400M;
1191         }
1192 }
1193
1194 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1195                                 struct vpu_session *session,
1196                                 void __user *src, u32 size)
1197 {
1198         struct vpu_service_info *pservice = data->pservice;
1199         int extra_size = 0;
1200         struct extra_info_for_iommu extra_info;
1201         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1202                                       GFP_KERNEL);
1203
1204         vpu_debug_enter();
1205
1206         if (!reg) {
1207                 vpu_err("error: kzalloc failed\n");
1208                 return NULL;
1209         }
1210
1211         if (size > data->reg_size) {
1212                 extra_size = size - data->reg_size;
1213                 size = data->reg_size;
1214         }
1215         reg->session = session;
1216         reg->data = data;
1217         reg->type = session->type;
1218         reg->size = size;
1219         reg->freq = VPU_FREQ_DEFAULT;
1220         reg->task = &data->task_info[session->type];
1221         reg->trans = data->trans_info;
1222         reg->reg = (u32 *)&reg[1];
1223         INIT_LIST_HEAD(&reg->session_link);
1224         INIT_LIST_HEAD(&reg->status_link);
1225
1226         INIT_LIST_HEAD(&reg->mem_region_list);
1227
1228         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1229                 vpu_err("error: copy_from_user failed\n");
1230                 kfree(reg);
1231                 return NULL;
1232         }
1233
1234         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1235                 vpu_err("error: copy_from_user failed\n");
1236                 kfree(reg);
1237                 return NULL;
1238         }
1239
1240         if (vcodec_reg_address_translate(data, session, reg, &extra_info) < 0) {
1241                 int i = 0;
1242
1243                 vpu_err("error: translate reg address failed, dumping regs\n");
1244                 for (i = 0; i < size >> 2; i++)
1245                         dev_err(pservice->dev, "reg[%02d]: %08x\n",
1246                                 i, *((u32 *)src + i));
1247
1248                 kfree(reg);
1249                 return NULL;
1250         }
1251
1252         mutex_lock(&pservice->lock);
1253         list_add_tail(&reg->status_link, &pservice->waiting);
1254         list_add_tail(&reg->session_link, &session->waiting);
1255         mutex_unlock(&pservice->lock);
1256
1257         if (pservice->auto_freq)
1258                 get_reg_freq(data, reg);
1259
1260         vpu_debug_leave();
1261
1262         return reg;
1263 }
1264
1265 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1266 {
1267         struct vpu_service_info *pservice = data->pservice;
1268         struct vcodec_mem_region *mem_region = NULL, *n;
1269
1270         list_del_init(&reg->session_link);
1271         list_del_init(&reg->status_link);
1272         if (reg == pservice->reg_codec)
1273                 pservice->reg_codec = NULL;
1274         if (reg == pservice->reg_pproc)
1275                 pservice->reg_pproc = NULL;
1276
1277         /* release memory region attach to this registers table. */
1278         list_for_each_entry_safe(mem_region, n,
1279                         &reg->mem_region_list, reg_lnk) {
1280                 vcodec_iommu_unmap_iommu(data->iommu_info, reg->session,
1281                                          mem_region->hdl);
1282                 vcodec_iommu_free(data->iommu_info, reg->session,
1283                                   mem_region->hdl);
1284                 list_del_init(&mem_region->reg_lnk);
1285                 kfree(mem_region);
1286         }
1287
1288         kfree(reg);
1289 }
1290
1291 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1292                                  struct vpu_reg *reg)
1293 {
1294         vpu_debug_enter();
1295         list_del_init(&reg->status_link);
1296         list_add_tail(&reg->status_link, &pservice->running);
1297
1298         list_del_init(&reg->session_link);
1299         list_add_tail(&reg->session_link, &reg->session->running);
1300         vpu_debug_leave();
1301 }
1302
1303 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1304 {
1305         int i;
1306         u32 *dst = reg->reg;
1307
1308         vpu_debug_enter();
1309         for (i = 0; i < count; i++, src++)
1310                 *dst++ = readl_relaxed(src);
1311
1312         dst = (u32 *)&reg->reg[0];
1313         for (i = 0; i < count; i++)
1314                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1315
1316         vpu_debug_leave();
1317 }
1318
1319 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1320                                  struct vpu_reg *reg)
1321 {
1322         struct vpu_service_info *pservice = data->pservice;
1323         struct vpu_hw_info *hw_info = data->hw_info;
1324         struct vpu_task_info *task = reg->task;
1325
1326         vpu_debug_enter();
1327
1328         list_del_init(&reg->status_link);
1329         list_add_tail(&reg->status_link, &pservice->done);
1330
1331         list_del_init(&reg->session_link);
1332         list_add_tail(&reg->session_link, &reg->session->done);
1333
1334         switch (reg->type) {
1335         case VPU_ENC: {
1336                 pservice->reg_codec = NULL;
1337                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1338                 reg->reg[task->reg_irq] = pservice->irq_status;
1339         } break;
1340         case VPU_DEC: {
1341                 pservice->reg_codec = NULL;
1342                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1343
1344                 /* revert hack for decoded length */
1345                 if (task->reg_len > 0) {
1346                         int reg_len = task->reg_len;
1347                         u32 dec_get = reg->reg[reg_len];
1348                         s32 dec_length = dec_get - reg->dec_base;
1349
1350                         vpu_debug(DEBUG_REGISTER,
1351                                   "dec_get %08x dec_length %d\n",
1352                                   dec_get, dec_length);
1353                         reg->reg[reg_len] = dec_length << 10;
1354                 }
1355
1356                 reg->reg[task->reg_irq] = pservice->irq_status;
1357         } break;
1358         case VPU_PP: {
1359                 pservice->reg_pproc = NULL;
1360                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1361                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1362         } break;
1363         case VPU_DEC_PP: {
1364                 u32 pipe_mode;
1365                 u32 *regs = data->dec_dev.regs;
1366
1367                 pservice->reg_codec = NULL;
1368                 pservice->reg_pproc = NULL;
1369
1370                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1371
1372                 /* NOTE: remove pp pipeline mode flag first */
1373                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1374                 pipe_mode &= ~task->pipe_mask;
1375                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1376
1377                 /* revert hack for decoded length */
1378                 if (task->reg_len > 0) {
1379                         int reg_len = task->reg_len;
1380                         u32 dec_get = reg->reg[reg_len];
1381                         s32 dec_length = dec_get - reg->dec_base;
1382
1383                         vpu_debug(DEBUG_REGISTER,
1384                                   "dec_get %08x dec_length %d\n",
1385                                   dec_get, dec_length);
1386                         reg->reg[reg_len] = dec_length << 10;
1387                 }
1388
1389                 reg->reg[task->reg_irq] = pservice->irq_status;
1390         } break;
1391         default: {
1392                 vpu_err("error: copy reg from hw with unknown type %d\n",
1393                         reg->type);
1394         } break;
1395         }
1396         vcodec_exit_mode(data);
1397
1398         atomic_sub(1, &reg->session->task_running);
1399         atomic_sub(1, &pservice->total_running);
1400         wake_up(&reg->session->wait);
1401
1402         vpu_debug_leave();
1403 }
1404
1405 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1406                                  struct vpu_reg *reg)
1407 {
1408         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1409
1410         if (curr == reg->freq)
1411                 return;
1412
1413         atomic_set(&pservice->freq_status, reg->freq);
1414         switch (reg->freq) {
1415         case VPU_FREQ_200M: {
1416                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1417         } break;
1418         case VPU_FREQ_266M: {
1419                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1420         } break;
1421         case VPU_FREQ_300M: {
1422                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1423         } break;
1424         case VPU_FREQ_400M: {
1425                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1426         } break;
1427         case VPU_FREQ_500M: {
1428                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1429         } break;
1430         case VPU_FREQ_600M: {
1431                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1432         } break;
1433         default: {
1434                 unsigned long rate = 300*MHZ;
1435
1436                 if (of_machine_is_compatible("rockchip,rk2928g"))
1437                         rate = 400*MHZ;
1438
1439                 clk_set_rate(pservice->aclk_vcodec, rate);
1440         } break;
1441         }
1442 }
1443
1444 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1445 {
1446         struct vpu_service_info *pservice = data->pservice;
1447         struct vpu_task_info *task = reg->task;
1448         struct vpu_hw_info *hw_info = data->hw_info;
1449         int i;
1450         u32 *src = (u32 *)&reg->reg[0];
1451         u32 enable_mask = task->enable_mask;
1452         u32 gating_mask = task->gating_mask;
1453         u32 reg_en = task->reg_en;
1454
1455         vpu_debug_enter();
1456
1457         atomic_add(1, &pservice->total_running);
1458         atomic_add(1, &reg->session->task_running);
1459
1460         if (pservice->auto_freq)
1461                 vpu_service_set_freq(pservice, reg);
1462
1463         vcodec_enter_mode(data);
1464
1465         switch (reg->type) {
1466         case VPU_ENC: {
1467                 u32 *dst = data->enc_dev.regs;
1468                 u32 base = 0;
1469                 u32 end  = hw_info->enc_reg_num;
1470                 /* u32 reg_gating = task->reg_gating; */
1471
1472                 pservice->reg_codec = reg;
1473
1474                 vpu_debug(DEBUG_TASK_INFO,
1475                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1476                           base, end, reg_en, enable_mask, gating_mask);
1477
1478                 VEPU_CLEAN_CACHE(dst);
1479
1480                 if (debug & DEBUG_SET_REG)
1481                         for (i = base; i < end; i++)
1482                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1483                                           i, src[i]);
1484
1485                 /*
1486                  * NOTE: encoder need to setup mode first
1487                  */
1488                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1489
1490                 /* NOTE: encoder gating is not on enable register */
1491                 /* src[reg_gating] |= gating_mask; */
1492
1493                 for (i = base; i < end; i++) {
1494                         if (i != reg_en)
1495                                 writel_relaxed(src[i], dst + i);
1496                 }
1497
1498                 writel(src[reg_en], dst + reg_en);
1499                 dsb(sy);
1500
1501                 time_record(reg->task, 0);
1502         } break;
1503         case VPU_DEC: {
1504                 u32 *dst = data->dec_dev.regs;
1505                 u32 len = hw_info->dec_reg_num;
1506                 u32 base = hw_info->base_dec;
1507                 u32 end  = hw_info->end_dec;
1508
1509                 pservice->reg_codec = reg;
1510
1511                 vpu_debug(DEBUG_TASK_INFO,
1512                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1513                           base, end, reg_en, enable_mask, gating_mask);
1514
1515                 VDPU_CLEAN_CACHE(dst);
1516
1517                 /* on rkvdec set cache size to 64byte */
1518                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1519                         u32 *cache_base = dst + 0x100;
1520                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1521                         writel_relaxed(val, cache_base + 0x07);
1522                         writel_relaxed(val, cache_base + 0x17);
1523                 }
1524
1525                 if (debug & DEBUG_SET_REG)
1526                         for (i = 0; i < len; i++)
1527                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1528                                           i, src[i]);
1529                 /*
1530                  * NOTE: The end register is invalid. Do NOT write to it
1531                  *       Also the base register must be written
1532                  */
1533                 for (i = base; i < end; i++) {
1534                         if (i != reg_en)
1535                                 writel_relaxed(src[i], dst + i);
1536                 }
1537
1538                 writel(src[reg_en] | gating_mask, dst + reg_en);
1539                 dsb(sy);
1540
1541                 time_record(reg->task, 0);
1542         } break;
1543         case VPU_PP: {
1544                 u32 *dst = data->dec_dev.regs;
1545                 u32 base = hw_info->base_pp;
1546                 u32 end  = hw_info->end_pp;
1547
1548                 pservice->reg_pproc = reg;
1549
1550                 vpu_debug(DEBUG_TASK_INFO,
1551                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1552                           base, end, reg_en, enable_mask, gating_mask);
1553
1554                 if (debug & DEBUG_SET_REG)
1555                         for (i = base; i < end; i++)
1556                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1557                                           i, src[i]);
1558
1559                 for (i = base; i < end; i++) {
1560                         if (i != reg_en)
1561                                 writel_relaxed(src[i], dst + i);
1562                 }
1563
1564                 writel(src[reg_en] | gating_mask, dst + reg_en);
1565                 dsb(sy);
1566
1567                 time_record(reg->task, 0);
1568         } break;
1569         case VPU_DEC_PP: {
1570                 u32 *dst = data->dec_dev.regs;
1571                 u32 base = hw_info->base_dec_pp;
1572                 u32 end  = hw_info->end_dec_pp;
1573
1574                 pservice->reg_codec = reg;
1575                 pservice->reg_pproc = reg;
1576
1577                 vpu_debug(DEBUG_TASK_INFO,
1578                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1579                           base, end, reg_en, enable_mask, gating_mask);
1580
1581                 /* VDPU_SOFT_RESET(dst); */
1582                 VDPU_CLEAN_CACHE(dst);
1583
1584                 if (debug & DEBUG_SET_REG)
1585                         for (i = base; i < end; i++)
1586                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1587                                           i, src[i]);
1588
1589                 for (i = base; i < end; i++) {
1590                         if (i != reg_en)
1591                                 writel_relaxed(src[i], dst + i);
1592                 }
1593
1594                 /* NOTE: dec output must be disabled */
1595
1596                 writel(src[reg_en] | gating_mask, dst + reg_en);
1597                 dsb(sy);
1598
1599                 time_record(reg->task, 0);
1600         } break;
1601         default: {
1602                 vpu_err("error: unsupport session type %d", reg->type);
1603                 atomic_sub(1, &pservice->total_running);
1604                 atomic_sub(1, &reg->session->task_running);
1605         } break;
1606         }
1607
1608         vpu_debug_leave();
1609 }
1610
1611 static void try_set_reg(struct vpu_subdev_data *data)
1612 {
1613         struct vpu_service_info *pservice = data->pservice;
1614
1615         vpu_debug_enter();
1616
1617         mutex_lock(&pservice->shutdown_lock);
1618         if (atomic_read(&pservice->service_on) == 0) {
1619                 mutex_unlock(&pservice->shutdown_lock);
1620                 return;
1621         }
1622         if (!list_empty(&pservice->waiting)) {
1623                 struct vpu_reg *reg_codec = pservice->reg_codec;
1624                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1625                 int can_set = 0;
1626                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1627                 int reset_request = atomic_read(&pservice->reset_request);
1628                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1629                                 struct vpu_reg, status_link);
1630
1631                 vpu_service_power_on(data, pservice);
1632
1633                 if (change_able || !reset_request) {
1634                         switch (reg->type) {
1635                         case VPU_ENC: {
1636                                 if (change_able)
1637                                         can_set = 1;
1638                         } break;
1639                         case VPU_DEC: {
1640                                 if (reg_codec == NULL)
1641                                         can_set = 1;
1642                                 if (pservice->auto_freq && (reg_pproc != NULL))
1643                                         can_set = 0;
1644                         } break;
1645                         case VPU_PP: {
1646                                 if (reg_codec == NULL) {
1647                                         if (reg_pproc == NULL)
1648                                                 can_set = 1;
1649                                 } else {
1650                                         if ((reg_codec->type == VPU_DEC) &&
1651                                             (reg_pproc == NULL))
1652                                                 can_set = 1;
1653
1654                                         /*
1655                                          * NOTE:
1656                                          * can not charge frequency
1657                                          * when vpu is working
1658                                          */
1659                                         if (pservice->auto_freq)
1660                                                 can_set = 0;
1661                                 }
1662                         } break;
1663                         case VPU_DEC_PP: {
1664                                 if (change_able)
1665                                         can_set = 1;
1666                                 } break;
1667                         default: {
1668                                 dev_err(pservice->dev,
1669                                         "undefined reg type %d\n",
1670                                         reg->type);
1671                         } break;
1672                         }
1673                 }
1674
1675                 /* then check reset request */
1676                 if (reset_request && !change_able)
1677                         reset_request = 0;
1678
1679                 /* do reset before setting registers */
1680                 if (reset_request)
1681                         vpu_reset(data);
1682
1683                 if (can_set) {
1684                         reg_from_wait_to_run(pservice, reg);
1685                         reg_copy_to_hw(reg->data, reg);
1686                 }
1687         }
1688
1689         mutex_unlock(&pservice->shutdown_lock);
1690         vpu_debug_leave();
1691 }
1692
1693 static void vpu_set_register_work(struct work_struct *work_s)
1694 {
1695         struct vpu_subdev_data *data = container_of(work_s,
1696                                                     struct vpu_subdev_data,
1697                                                     set_work);
1698         struct vpu_service_info *pservice = data->pservice;
1699
1700         mutex_lock(&pservice->lock);
1701         try_set_reg(data);
1702         mutex_unlock(&pservice->lock);
1703 }
1704
1705 static int return_reg(struct vpu_subdev_data *data,
1706                       struct vpu_reg *reg, u32 __user *dst)
1707 {
1708         struct vpu_hw_info *hw_info = data->hw_info;
1709         size_t size = reg->size;
1710         u32 base;
1711
1712         vpu_debug_enter();
1713         switch (reg->type) {
1714         case VPU_ENC: {
1715                 base = 0;
1716         } break;
1717         case VPU_DEC: {
1718                 base = hw_info->base_dec_pp;
1719         } break;
1720         case VPU_PP: {
1721                 base = hw_info->base_pp;
1722         } break;
1723         case VPU_DEC_PP: {
1724                 base = hw_info->base_dec_pp;
1725         } break;
1726         default: {
1727                 vpu_err("error: copy reg to user with unknown type %d\n",
1728                         reg->type);
1729                 return -EFAULT;
1730         } break;
1731         }
1732
1733         if (copy_to_user(dst, &reg->reg[base], size)) {
1734                 vpu_err("error: copy_to_user failed\n");
1735                 return -EFAULT;
1736         }
1737
1738         reg_deinit(data, reg);
1739         vpu_debug_leave();
1740         return 0;
1741 }
1742
1743 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1744                               unsigned long arg)
1745 {
1746         struct vpu_subdev_data *data =
1747                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1748                              struct vpu_subdev_data, cdev);
1749         struct vpu_service_info *pservice = data->pservice;
1750         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1751
1752         vpu_debug_enter();
1753         if (NULL == session)
1754                 return -EINVAL;
1755
1756         switch (cmd) {
1757         case VPU_IOC_SET_CLIENT_TYPE: {
1758                 session->type = (enum VPU_CLIENT_TYPE)arg;
1759                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1760                           session->pid, session->type);
1761         } break;
1762         case VPU_IOC_GET_HW_FUSE_STATUS: {
1763                 struct vpu_request req;
1764
1765                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1766                           session->pid, session->type);
1767                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1768                         vpu_err("error: get hw status copy_from_user failed\n");
1769                         return -EFAULT;
1770                 } else {
1771                         void *config = (session->type != VPU_ENC) ?
1772                                        ((void *)&pservice->dec_config) :
1773                                        ((void *)&pservice->enc_config);
1774                         size_t size = (session->type != VPU_ENC) ?
1775                                       (sizeof(struct vpu_dec_config)) :
1776                                       (sizeof(struct vpu_enc_config));
1777                         if (copy_to_user((void __user *)req.req,
1778                                          config, size)) {
1779                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1780                                         session->type);
1781                                 return -EFAULT;
1782                         }
1783                 }
1784         } break;
1785         case VPU_IOC_SET_REG: {
1786                 struct vpu_request req;
1787                 struct vpu_reg *reg;
1788
1789                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1790                           session->pid, session->type);
1791                 if (copy_from_user(&req, (void __user *)arg,
1792                                    sizeof(struct vpu_request))) {
1793                         vpu_err("error: set reg copy_from_user failed\n");
1794                         return -EFAULT;
1795                 }
1796
1797                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1798                 if (NULL == reg) {
1799                         return -EFAULT;
1800                 } else {
1801                         queue_work(pservice->set_workq, &data->set_work);
1802                 }
1803         } break;
1804         case VPU_IOC_GET_REG: {
1805                 struct vpu_request req;
1806                 struct vpu_reg *reg;
1807                 int ret;
1808
1809                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1810                           session->pid, session->type);
1811                 if (copy_from_user(&req, (void __user *)arg,
1812                                    sizeof(struct vpu_request))) {
1813                         vpu_err("error: get reg copy_from_user failed\n");
1814                         return -EFAULT;
1815                 }
1816
1817                 ret = wait_event_timeout(session->wait,
1818                                          !list_empty(&session->done),
1819                                          VPU_TIMEOUT_DELAY);
1820
1821                 if (!list_empty(&session->done)) {
1822                         if (ret < 0)
1823                                 vpu_err("warning: pid %d wait task error ret %d\n",
1824                                         session->pid, ret);
1825                         ret = 0;
1826                 } else {
1827                         if (unlikely(ret < 0)) {
1828                                 vpu_err("error: pid %d wait task ret %d\n",
1829                                         session->pid, ret);
1830                         } else if (ret == 0) {
1831                                 vpu_err("error: pid %d wait %d task done timeout\n",
1832                                         session->pid,
1833                                         atomic_read(&session->task_running));
1834                                 ret = -ETIMEDOUT;
1835                         }
1836                 }
1837
1838                 if (ret < 0) {
1839                         int task_running = atomic_read(&session->task_running);
1840
1841                         mutex_lock(&pservice->lock);
1842                         vpu_service_dump(pservice);
1843                         if (task_running) {
1844                                 atomic_set(&session->task_running, 0);
1845                                 atomic_sub(task_running,
1846                                            &pservice->total_running);
1847                                 dev_err(pservice->dev,
1848                                         "%d task is running but not return, reset hardware...",
1849                                         task_running);
1850                                 vpu_reset(data);
1851                                 dev_err(pservice->dev, "done\n");
1852                         }
1853                         vpu_service_session_clear(data, session);
1854                         mutex_unlock(&pservice->lock);
1855
1856                         return ret;
1857                 }
1858                 mutex_lock(&pservice->lock);
1859                 reg = list_entry(session->done.next,
1860                                  struct vpu_reg, session_link);
1861                 return_reg(data, reg, (u32 __user *)req.req);
1862                 mutex_unlock(&pservice->lock);
1863         } break;
1864         case VPU_IOC_PROBE_IOMMU_STATUS: {
1865                 int iommu_enable = 1;
1866
1867                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1868
1869                 if (copy_to_user((void __user *)arg,
1870                                  &iommu_enable, sizeof(int))) {
1871                         vpu_err("error: iommu status copy_to_user failed\n");
1872                         return -EFAULT;
1873                 }
1874         } break;
1875         default: {
1876                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1877         } break;
1878         }
1879         vpu_debug_leave();
1880         return 0;
1881 }
1882
1883 #ifdef CONFIG_COMPAT
1884 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1885                                      unsigned long arg)
1886 {
1887         struct vpu_subdev_data *data =
1888                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1889                              struct vpu_subdev_data, cdev);
1890         struct vpu_service_info *pservice = data->pservice;
1891         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1892
1893         vpu_debug_enter();
1894         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1895                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1896         if (NULL == session)
1897                 return -EINVAL;
1898
1899         switch (cmd) {
1900         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1901                 session->type = (enum VPU_CLIENT_TYPE)arg;
1902                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1903                           session->type);
1904         } break;
1905         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1906                 struct compat_vpu_request req;
1907
1908                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1909                           session->type);
1910                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1911                                    sizeof(struct compat_vpu_request))) {
1912                         vpu_err("error: compat get hw status copy_from_user failed\n");
1913                         return -EFAULT;
1914                 } else {
1915                         void *config = (session->type != VPU_ENC) ?
1916                                        ((void *)&pservice->dec_config) :
1917                                        ((void *)&pservice->enc_config);
1918                         size_t size = (session->type != VPU_ENC) ?
1919                                       (sizeof(struct vpu_dec_config)) :
1920                                       (sizeof(struct vpu_enc_config));
1921
1922                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1923                                          config, size)) {
1924                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1925                                         session->type);
1926                                 return -EFAULT;
1927                         }
1928                 }
1929         } break;
1930         case COMPAT_VPU_IOC_SET_REG: {
1931                 struct compat_vpu_request req;
1932                 struct vpu_reg *reg;
1933
1934                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1935                           session->type);
1936                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1937                                    sizeof(struct compat_vpu_request))) {
1938                         vpu_err("compat set_reg copy_from_user failed\n");
1939                         return -EFAULT;
1940                 }
1941                 reg = reg_init(data, session,
1942                                compat_ptr((compat_uptr_t)req.req), req.size);
1943                 if (NULL == reg) {
1944                         return -EFAULT;
1945                 } else {
1946                         queue_work(pservice->set_workq, &data->set_work);
1947                 }
1948         } break;
1949         case COMPAT_VPU_IOC_GET_REG: {
1950                 struct compat_vpu_request req;
1951                 struct vpu_reg *reg;
1952                 int ret;
1953
1954                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1955                           session->type);
1956                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1957                                    sizeof(struct compat_vpu_request))) {
1958                         vpu_err("compat get reg copy_from_user failed\n");
1959                         return -EFAULT;
1960                 }
1961
1962                 ret = wait_event_timeout(session->wait,
1963                                          !list_empty(&session->done),
1964                                          VPU_TIMEOUT_DELAY);
1965
1966                 if (!list_empty(&session->done)) {
1967                         if (ret < 0)
1968                                 vpu_err("warning: pid %d wait task error ret %d\n",
1969                                         session->pid, ret);
1970                         ret = 0;
1971                 } else {
1972                         if (unlikely(ret < 0)) {
1973                                 vpu_err("error: pid %d wait task ret %d\n",
1974                                         session->pid, ret);
1975                         } else if (ret == 0) {
1976                                 vpu_err("error: pid %d wait %d task done timeout\n",
1977                                         session->pid,
1978                                         atomic_read(&session->task_running));
1979                                 ret = -ETIMEDOUT;
1980                         }
1981                 }
1982
1983                 if (ret < 0) {
1984                         int task_running = atomic_read(&session->task_running);
1985
1986                         mutex_lock(&pservice->lock);
1987                         vpu_service_dump(pservice);
1988                         if (task_running) {
1989                                 atomic_set(&session->task_running, 0);
1990                                 atomic_sub(task_running,
1991                                            &pservice->total_running);
1992                                 dev_err(pservice->dev,
1993                                         "%d task is running but not return, reset hardware...",
1994                                         task_running);
1995                                 vpu_reset(data);
1996                                 dev_err(pservice->dev, "done\n");
1997                         }
1998                         vpu_service_session_clear(data, session);
1999                         mutex_unlock(&pservice->lock);
2000                         return ret;
2001                 }
2002
2003                 mutex_lock(&pservice->lock);
2004                 reg = list_entry(session->done.next,
2005                                  struct vpu_reg, session_link);
2006                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
2007                 mutex_unlock(&pservice->lock);
2008         } break;
2009         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
2010                 int iommu_enable = 1;
2011
2012                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
2013
2014                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
2015                                  &iommu_enable, sizeof(int))) {
2016                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2017                         return -EFAULT;
2018                 }
2019         } break;
2020         default: {
2021                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2022         } break;
2023         }
2024         vpu_debug_leave();
2025         return 0;
2026 }
2027 #endif
2028
2029 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2030 {
2031         struct vpu_service_info *pservice = data->pservice;
2032         int ret = -EINVAL, i = 0;
2033         u32 hw_id = readl_relaxed(data->regs);
2034
2035         hw_id = (hw_id >> 16) & 0xFFFF;
2036         dev_info(pservice->dev, "checking hw id %x\n", hw_id);
2037         data->hw_info = NULL;
2038
2039         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2040                 const struct vcodec_info *info = &vcodec_info_set[i];
2041
2042                 if (hw_id == info->hw_id) {
2043                         data->hw_id = info->hw_id;
2044                         data->hw_info = info->hw_info;
2045                         data->task_info = info->task_info;
2046                         data->trans_info = info->trans_info;
2047                         ret = 0;
2048                         break;
2049                 }
2050         }
2051         return ret;
2052 }
2053
2054 static int vpu_service_open(struct inode *inode, struct file *filp)
2055 {
2056         struct vpu_subdev_data *data = container_of(
2057                         inode->i_cdev, struct vpu_subdev_data, cdev);
2058         struct vpu_service_info *pservice = data->pservice;
2059         struct vpu_session *session = NULL;
2060
2061         vpu_debug_enter();
2062
2063         session = kzalloc(sizeof(*session), GFP_KERNEL);
2064         if (!session) {
2065                 vpu_err("error: unable to allocate memory for vpu_session.");
2066                 return -ENOMEM;
2067         }
2068
2069         data->iommu_info->debug_level = debug;
2070
2071         session->type   = VPU_TYPE_BUTT;
2072         session->pid    = current->pid;
2073         INIT_LIST_HEAD(&session->waiting);
2074         INIT_LIST_HEAD(&session->running);
2075         INIT_LIST_HEAD(&session->done);
2076         INIT_LIST_HEAD(&session->list_session);
2077         init_waitqueue_head(&session->wait);
2078         atomic_set(&session->task_running, 0);
2079         mutex_lock(&pservice->lock);
2080         list_add_tail(&session->list_session, &pservice->session);
2081         filp->private_data = (void *)session;
2082         mutex_unlock(&pservice->lock);
2083
2084         dev_dbg(pservice->dev, "dev opened\n");
2085         vpu_debug_leave();
2086         return nonseekable_open(inode, filp);
2087 }
2088
2089 static int vpu_service_release(struct inode *inode, struct file *filp)
2090 {
2091         struct vpu_subdev_data *data = container_of(
2092                         inode->i_cdev, struct vpu_subdev_data, cdev);
2093         struct vpu_service_info *pservice = data->pservice;
2094         int task_running;
2095         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2096
2097         vpu_debug_enter();
2098         if (NULL == session)
2099                 return -EINVAL;
2100
2101         task_running = atomic_read(&session->task_running);
2102         if (task_running) {
2103                 dev_err(pservice->dev,
2104                         "error: session %d still has %d task running when closing\n",
2105                         session->pid, task_running);
2106                 msleep(50);
2107         }
2108         wake_up(&session->wait);
2109
2110         mutex_lock(&pservice->lock);
2111         /* remove this filp from the asynchronusly notified filp's */
2112         list_del_init(&session->list_session);
2113         vpu_service_session_clear(data, session);
2114         vcodec_iommu_clear(data->iommu_info, session);
2115         kfree(session);
2116         filp->private_data = NULL;
2117         mutex_unlock(&pservice->lock);
2118
2119         dev_info(pservice->dev, "closed\n");
2120         vpu_debug_leave();
2121         return 0;
2122 }
2123
2124 static const struct file_operations vpu_service_fops = {
2125         .unlocked_ioctl = vpu_service_ioctl,
2126         .open           = vpu_service_open,
2127         .release        = vpu_service_release,
2128 #ifdef CONFIG_COMPAT
2129         .compat_ioctl   = compat_vpu_service_ioctl,
2130 #endif
2131 };
2132
2133 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2134 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2135 static irqreturn_t vepu_irq(int irq, void *dev_id);
2136 static irqreturn_t vepu_isr(int irq, void *dev_id);
2137 static void get_hw_info(struct vpu_subdev_data *data);
2138
2139 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2140 {
2141         struct device_node *dn = NULL;
2142         struct platform_device *pd = NULL;
2143         struct device *ret = NULL;
2144
2145         dn = of_find_compatible_node(NULL, NULL, compt);
2146         if (!dn) {
2147                 pr_err("can't find device node %s \r\n", compt);
2148                 return NULL;
2149         }
2150
2151         pd = of_find_device_by_node(dn);
2152         if (!pd) {
2153                 pr_err("can't find platform device in device node %s\n", compt);
2154                 return  NULL;
2155         }
2156         ret = &pd->dev;
2157
2158         return ret;
2159 }
2160
2161 #ifdef CONFIG_IOMMU_API
2162 static inline void platform_set_sysmmu(struct device *iommu,
2163                                        struct device *dev)
2164 {
2165         dev->archdata.iommu = iommu;
2166 }
2167 #else
2168 static inline void platform_set_sysmmu(struct device *iommu,
2169                                        struct device *dev)
2170 {
2171 }
2172 #endif
2173
2174 int vcodec_sysmmu_fault_hdl(struct device *dev,
2175                             enum rk_iommu_inttype itype,
2176                             unsigned long pgtable_base,
2177                             unsigned long fault_addr, unsigned int status)
2178 {
2179         struct platform_device *pdev;
2180         struct vpu_service_info *pservice;
2181         struct vpu_subdev_data *data;
2182
2183         vpu_debug_enter();
2184
2185         if (dev == NULL) {
2186                 pr_err("invalid NULL dev\n");
2187                 return 0;
2188         }
2189
2190         pdev = container_of(dev, struct platform_device, dev);
2191         if (pdev == NULL) {
2192                 pr_err("invalid NULL platform_device\n");
2193                 return 0;
2194         }
2195
2196         data = platform_get_drvdata(pdev);
2197         if (data == NULL) {
2198                 pr_err("invalid NULL vpu_subdev_data\n");
2199                 return 0;
2200         }
2201
2202         pservice = data->pservice;
2203         if (pservice == NULL) {
2204                 pr_err("invalid NULL vpu_service_info\n");
2205                 return 0;
2206         }
2207
2208         if (pservice->reg_codec) {
2209                 struct vpu_reg *reg = pservice->reg_codec;
2210                 struct vcodec_mem_region *mem, *n;
2211                 int i = 0;
2212
2213                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2214                 if (!list_empty(&reg->mem_region_list)) {
2215                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2216                                                  reg_lnk) {
2217                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2218                                        mem->reg_idx, i, mem->iova, mem->len);
2219                                 i++;
2220                         }
2221                 } else {
2222                         pr_err("no memory region mapped\n");
2223                 }
2224
2225                 if (reg->data) {
2226                         struct vpu_subdev_data *data = reg->data;
2227                         u32 *base = (u32 *)data->dec_dev.regs;
2228                         u32 len = data->hw_info->dec_reg_num;
2229
2230                         pr_err("current errror register set:\n");
2231
2232                         for (i = 0; i < len; i++)
2233                                 pr_err("reg[%02d] %08x\n",
2234                                        i, readl_relaxed(base + i));
2235                 }
2236
2237                 pr_alert("vcodec, page fault occur, reset hw\n");
2238
2239                 /* reg->reg[101] = 1; */
2240                 _vpu_reset(data);
2241         }
2242
2243         return 0;
2244 }
2245
2246 static int vcodec_subdev_probe(struct platform_device *pdev,
2247                                struct vpu_service_info *pservice)
2248 {
2249         uint8_t *regs = NULL;
2250         int32_t ret = 0;
2251         uint32_t ioaddr = 0;
2252         struct resource *res = NULL;
2253         struct vpu_hw_info *hw_info = NULL;
2254         struct device *dev = &pdev->dev;
2255         struct device_node *np = pdev->dev.of_node;
2256         struct vpu_subdev_data *data = NULL;
2257         struct platform_device *sub_dev = NULL;
2258         struct device_node *sub_np = NULL;
2259         const char *name  = np->name;
2260         char mmu_dev_dts_name[40];
2261
2262         dev_info(dev, "probe device");
2263
2264         data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2265         if (!data)
2266                 return -ENOMEM;
2267
2268         data->pservice = pservice;
2269         data->dev = dev;
2270
2271         INIT_WORK(&data->set_work, vpu_set_register_work);
2272         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2273
2274         if (pservice->reg_base == 0) {
2275                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2276                 data->regs = devm_ioremap_resource(dev, res);
2277                 if (IS_ERR(data->regs)) {
2278                         ret = PTR_ERR(data->regs);
2279                         goto err;
2280                 }
2281                 ioaddr = res->start;
2282         } else {
2283                 data->regs = pservice->reg_base;
2284                 ioaddr = pservice->ioaddr;
2285         }
2286
2287         sub_np = of_parse_phandle(np, "iommus", 0);
2288         if (sub_np) {
2289                 sub_dev = of_find_device_by_node(sub_np);
2290                 data->mmu_dev = &sub_dev->dev;
2291         }
2292
2293         /* Back to legacy iommu probe */
2294         if (!data->mmu_dev) {
2295                 switch (data->mode) {
2296                 case VCODEC_RUNNING_MODE_VPU:
2297                         sprintf(mmu_dev_dts_name,
2298                                 VPU_IOMMU_COMPATIBLE_NAME);
2299                         break;
2300                 case VCODEC_RUNNING_MODE_RKVDEC:
2301                         sprintf(mmu_dev_dts_name,
2302                                 VDEC_IOMMU_COMPATIBLE_NAME);
2303                         break;
2304                 case VCODEC_RUNNING_MODE_HEVC:
2305                 default:
2306                         sprintf(mmu_dev_dts_name,
2307                                 HEVC_IOMMU_COMPATIBLE_NAME);
2308                         break;
2309                 }
2310
2311                 data->mmu_dev =
2312                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2313                 if (data->mmu_dev)
2314                         platform_set_sysmmu(data->mmu_dev, dev);
2315
2316                 rockchip_iovmm_set_fault_handler
2317                         (dev, vcodec_sysmmu_fault_hdl);
2318         }
2319
2320         dev_info(dev, "vpu mmu dec %p\n", data->mmu_dev);
2321
2322         clear_bit(MMU_ACTIVATED, &data->state);
2323         vpu_service_power_on(data, pservice);
2324
2325         of_property_read_u32(np, "allocator", (u32 *)&pservice->alloc_type);
2326         data->iommu_info = vcodec_iommu_info_create(dev, data->mmu_dev,
2327                                                     pservice->alloc_type);
2328         dev_info(dev, "allocator is %s\n", pservice->alloc_type == 1 ? "drm" :
2329                 (pservice->alloc_type == 2 ? "ion" : "null"));
2330         vcodec_enter_mode(data);
2331         ret = vpu_service_check_hw(data);
2332         if (ret < 0) {
2333                 vpu_err("error: hw info check faild\n");
2334                 goto err;
2335         }
2336         vcodec_exit_mode(data);
2337
2338         hw_info = data->hw_info;
2339         regs = (u8 *)data->regs;
2340
2341         if (hw_info->dec_reg_num) {
2342                 data->dec_dev.iosize = hw_info->dec_io_size;
2343                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2344         }
2345
2346         if (hw_info->enc_reg_num) {
2347                 data->enc_dev.iosize = hw_info->enc_io_size;
2348                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2349         }
2350
2351         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2352
2353         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2354         if (data->irq_enc > 0) {
2355                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2356                                                 vepu_irq, vepu_isr,
2357                                                 IRQF_SHARED, dev_name(dev),
2358                                                 (void *)data);
2359                 if (ret) {
2360                         dev_err(dev, "error: can't request vepu irq %d\n",
2361                                 data->irq_enc);
2362                         goto err;
2363                 }
2364         }
2365         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2366         if (data->irq_dec > 0) {
2367                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2368                                                 vdpu_irq, vdpu_isr,
2369                                                 IRQF_SHARED, dev_name(dev),
2370                                                 (void *)data);
2371                 if (ret) {
2372                         dev_err(dev, "error: can't request vdpu irq %d\n",
2373                                 data->irq_dec);
2374                         goto err;
2375                 }
2376         }
2377         atomic_set(&data->dec_dev.irq_count_codec, 0);
2378         atomic_set(&data->dec_dev.irq_count_pp, 0);
2379         atomic_set(&data->enc_dev.irq_count_codec, 0);
2380         atomic_set(&data->enc_dev.irq_count_pp, 0);
2381
2382         get_hw_info(data);
2383         pservice->auto_freq = true;
2384
2385         /* create device node */
2386         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2387         if (ret) {
2388                 dev_err(dev, "alloc dev_t failed\n");
2389                 goto err;
2390         }
2391
2392         cdev_init(&data->cdev, &vpu_service_fops);
2393
2394         data->cdev.owner = THIS_MODULE;
2395         data->cdev.ops = &vpu_service_fops;
2396
2397         ret = cdev_add(&data->cdev, data->dev_t, 1);
2398
2399         if (ret) {
2400                 dev_err(dev, "add dev_t failed\n");
2401                 goto err;
2402         }
2403
2404         data->cls = class_create(THIS_MODULE, name);
2405
2406         if (IS_ERR(data->cls)) {
2407                 ret = PTR_ERR(data->cls);
2408                 dev_err(dev, "class_create err:%d\n", ret);
2409                 goto err;
2410         }
2411
2412         data->child_dev = device_create(data->cls, dev,
2413                 data->dev_t, "%s", name);
2414
2415         platform_set_drvdata(pdev, data);
2416
2417         INIT_LIST_HEAD(&data->lnk_service);
2418         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2419
2420         return 0;
2421 err:
2422         if (data->child_dev) {
2423                 device_destroy(data->cls, data->dev_t);
2424                 cdev_del(&data->cdev);
2425                 unregister_chrdev_region(data->dev_t, 1);
2426         }
2427
2428         if (data->cls)
2429                 class_destroy(data->cls);
2430         return -1;
2431 }
2432
2433 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2434 {
2435         struct vpu_service_info *pservice = data->pservice;
2436
2437         vcodec_iommu_info_destroy(data->iommu_info);
2438         data->iommu_info = NULL;
2439
2440         mutex_lock(&pservice->lock);
2441         cancel_delayed_work_sync(&pservice->power_off_work);
2442         vpu_service_power_off(pservice);
2443         mutex_unlock(&pservice->lock);
2444
2445         device_destroy(data->cls, data->dev_t);
2446         class_destroy(data->cls);
2447         cdev_del(&data->cdev);
2448         unregister_chrdev_region(data->dev_t, 1);
2449
2450 #ifdef CONFIG_DEBUG_FS
2451         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2452                 debugfs_remove_recursive(data->debugfs_dir);
2453 #endif
2454 }
2455
2456 static void vcodec_read_property(struct device_node *np,
2457                                  struct vpu_service_info *pservice)
2458 {
2459         pservice->mode_bit = 0;
2460         pservice->mode_ctrl = 0;
2461         pservice->subcnt = 0;
2462         pservice->grf_base = NULL;
2463
2464         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2465
2466         if (pservice->subcnt > 1) {
2467                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2468                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2469         }
2470 #ifdef CONFIG_MFD_SYSCON
2471         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2472         if (IS_ERR_OR_NULL(pservice->grf)) {
2473                 pservice->grf = NULL;
2474 #ifdef CONFIG_ARM
2475                 pservice->grf_base = RK_GRF_VIRT;
2476 #else
2477                 vpu_err("can't find vpu grf property\n");
2478                 return;
2479 #endif
2480         }
2481 #else
2482 #ifdef CONFIG_ARM
2483         pservice->grf_base = RK_GRF_VIRT;
2484 #else
2485         vpu_err("can't find vpu grf property\n");
2486         return;
2487 #endif
2488 #endif
2489
2490 #ifdef CONFIG_RESET_CONTROLLER
2491         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2492         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2493         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2494
2495         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2496                 dev_warn(pservice->dev, "No aclk reset resource define\n");
2497                 pservice->rst_a = NULL;
2498         }
2499
2500         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2501                 dev_warn(pservice->dev, "No hclk reset resource define\n");
2502                 pservice->rst_h = NULL;
2503         }
2504
2505         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2506                 dev_warn(pservice->dev, "No core reset resource define\n");
2507                 pservice->rst_v = NULL;
2508         }
2509 #endif
2510
2511         of_property_read_string(np, "name", (const char **)&pservice->name);
2512 }
2513
2514 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2515 {
2516         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2517         pservice->curr_mode = -1;
2518
2519         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2520         INIT_LIST_HEAD(&pservice->waiting);
2521         INIT_LIST_HEAD(&pservice->running);
2522         mutex_init(&pservice->lock);
2523         mutex_init(&pservice->shutdown_lock);
2524         atomic_set(&pservice->service_on, 1);
2525
2526         INIT_LIST_HEAD(&pservice->done);
2527         INIT_LIST_HEAD(&pservice->session);
2528         INIT_LIST_HEAD(&pservice->subdev_list);
2529
2530         pservice->reg_pproc     = NULL;
2531         atomic_set(&pservice->total_running, 0);
2532         atomic_set(&pservice->enabled,       0);
2533         atomic_set(&pservice->power_on_cnt,  0);
2534         atomic_set(&pservice->power_off_cnt, 0);
2535         atomic_set(&pservice->reset_request, 0);
2536
2537         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2538         pservice->last.tv64 = 0;
2539
2540         pservice->alloc_type = 0;
2541 }
2542
2543 static int vcodec_probe(struct platform_device *pdev)
2544 {
2545         int i;
2546         int ret = 0;
2547         struct resource *res = NULL;
2548         struct device *dev = &pdev->dev;
2549         struct device_node *np = pdev->dev.of_node;
2550         struct vpu_service_info *pservice = NULL;
2551         struct vcodec_device_info *driver_data;
2552
2553         pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info),
2554                                 GFP_KERNEL);
2555         if (!pservice)
2556                 return -ENOMEM;
2557         pservice->dev = dev;
2558
2559         pservice->set_workq = create_singlethread_workqueue("vcodec");
2560         if (!pservice->set_workq) {
2561                 dev_err(dev, "failed to create workqueue\n");
2562                 return -ENOMEM;
2563         }
2564
2565         driver_data = vcodec_get_drv_data(pdev);
2566         if (!driver_data)
2567                 return -EINVAL;
2568
2569         vcodec_read_property(np, pservice);
2570         vcodec_init_drvdata(pservice);
2571
2572         /* Underscore for label, hyphens for name */
2573         switch (driver_data->device_type) {
2574         case VCODEC_DEVICE_TYPE_VPUX:
2575                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2576                 break;
2577         case VCODEC_DEVICE_TYPE_VPUC:
2578                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2579                 break;
2580         case VCODEC_DEVICE_TYPE_HEVC:
2581                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2582                 break;
2583         case VCODEC_DEVICE_TYPE_RKVD:
2584                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2585                 break;
2586         default:
2587                 dev_err(dev, "unsupported device type\n");
2588                 return -ENODEV;
2589         }
2590
2591         if (0 > vpu_get_clk(pservice))
2592                 goto err;
2593
2594         if (of_property_read_bool(np, "reg")) {
2595                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2596
2597                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2598                 if (IS_ERR(pservice->reg_base)) {
2599                         vpu_err("ioremap registers base failed\n");
2600                         ret = PTR_ERR(pservice->reg_base);
2601                         goto err;
2602                 }
2603                 pservice->ioaddr = res->start;
2604         } else {
2605                 pservice->reg_base = 0;
2606         }
2607
2608         pm_runtime_enable(dev);
2609
2610         if (of_property_read_bool(np, "subcnt")) {
2611                 struct vpu_subdev_data *data = NULL;
2612
2613                 data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data),
2614                                     GFP_KERNEL);
2615                 if (!data)
2616                         return -ENOMEM;
2617
2618                 for (i = 0; i < pservice->subcnt; i++) {
2619                         struct device_node *sub_np;
2620                         struct platform_device *sub_pdev;
2621
2622                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2623                         sub_pdev = of_find_device_by_node(sub_np);
2624
2625                         vcodec_subdev_probe(sub_pdev, pservice);
2626                 }
2627                 data->pservice = pservice;
2628                 platform_set_drvdata(pdev, data);
2629         } else {
2630                 vcodec_subdev_probe(pdev, pservice);
2631         }
2632
2633         vpu_service_power_off(pservice);
2634
2635         dev_info(dev, "init success\n");
2636
2637         return 0;
2638
2639 err:
2640         dev_info(dev, "init failed\n");
2641         vpu_service_power_off(pservice);
2642         destroy_workqueue(pservice->set_workq);
2643         wake_lock_destroy(&pservice->wake_lock);
2644
2645         return ret;
2646 }
2647
2648 static int vcodec_remove(struct platform_device *pdev)
2649 {
2650         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2651
2652         vcodec_subdev_remove(data);
2653
2654         pm_runtime_disable(data->pservice->dev);
2655
2656         return 0;
2657 }
2658
2659 static void vcodec_shutdown(struct platform_device *pdev)
2660 {
2661         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2662         struct vpu_service_info *pservice = data->pservice;
2663         struct device_node *np = pdev->dev.of_node;
2664         int val;
2665         int ret;
2666         int i;
2667
2668         dev_info(&pdev->dev, "vcodec shutdown");
2669
2670         mutex_lock(&pservice->shutdown_lock);
2671         atomic_set(&pservice->service_on, 0);
2672         mutex_unlock(&pservice->shutdown_lock);
2673
2674         ret = readx_poll_timeout(atomic_read,
2675                                  &pservice->total_running,
2676                                  val, val == 0, 20000, 200000);
2677         if (ret == -ETIMEDOUT)
2678                 dev_err(&pdev->dev, "wait total running time out\n");
2679
2680         vcodec_exit_mode(data);
2681         vpu_service_clear(data);
2682         if (of_property_read_bool(np, "subcnt")) {
2683                 for (i = 0; i < pservice->subcnt; i++) {
2684                         struct device_node *sub_np;
2685                         struct platform_device *sub_pdev;
2686
2687                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2688                         sub_pdev = of_find_device_by_node(sub_np);
2689                         vcodec_subdev_remove(platform_get_drvdata(sub_pdev));
2690                 }
2691
2692         } else {
2693                 vcodec_subdev_remove(data);
2694         }
2695
2696         pm_runtime_disable(&pdev->dev);
2697 }
2698
2699 static const struct of_device_id vcodec_service_dt_ids[] = {
2700         {
2701                 .compatible = "rockchip,vpu_service",
2702                 .data = &vpu_device_info,
2703         },
2704         {
2705                 .compatible = "rockchip,hevc_service",
2706                 .data = &hevc_device_info,
2707         },
2708         {
2709                 .compatible = "rockchip,vpu_combo",
2710                 .data = &vpu_combo_device_info,
2711         },
2712         {
2713                 .compatible = "rockchip,rkvdec",
2714                 .data = &rkvd_device_info,
2715         },
2716         {},
2717 };
2718
2719 MODULE_DEVICE_TABLE(of, vcodec_service_dt_ids);
2720
2721 static void *vcodec_get_drv_data(struct platform_device *pdev)
2722 {
2723         struct vcodec_device_info *driver_data = NULL;
2724         const struct of_device_id *match;
2725
2726         match = of_match_node(vcodec_service_dt_ids, pdev->dev.of_node);
2727         if (match)
2728                 driver_data = (struct vcodec_device_info *)match->data;
2729
2730         return driver_data;
2731 }
2732
2733 static struct platform_driver vcodec_driver = {
2734         .probe = vcodec_probe,
2735         .remove = vcodec_remove,
2736         .shutdown = vcodec_shutdown,
2737         .driver = {
2738                 .name = "rk-vcodec",
2739                 .owner = THIS_MODULE,
2740                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2741         },
2742 };
2743
2744 static void get_hw_info(struct vpu_subdev_data *data)
2745 {
2746         struct vpu_service_info *pservice = data->pservice;
2747         struct vpu_dec_config *dec = &pservice->dec_config;
2748         struct vpu_enc_config *enc = &pservice->enc_config;
2749
2750         if (of_machine_is_compatible("rockchip,rk2928") ||
2751                         of_machine_is_compatible("rockchip,rk3036") ||
2752                         of_machine_is_compatible("rockchip,rk3066") ||
2753                         of_machine_is_compatible("rockchip,rk3126") ||
2754                         of_machine_is_compatible("rockchip,rk3188"))
2755                 dec->max_dec_pic_width = 1920;
2756         else
2757                 dec->max_dec_pic_width = 4096;
2758
2759         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2760                 dec->h264_support = 3;
2761                 dec->jpeg_support = 1;
2762                 dec->mpeg4_support = 2;
2763                 dec->vc1_support = 3;
2764                 dec->mpeg2_support = 1;
2765                 dec->pp_support = 1;
2766                 dec->sorenson_support = 1;
2767                 dec->ref_buf_support = 3;
2768                 dec->vp6_support = 1;
2769                 dec->vp7_support = 1;
2770                 dec->vp8_support = 1;
2771                 dec->avs_support = 1;
2772                 dec->jpeg_ext_support = 0;
2773                 dec->custom_mpeg4_support = 1;
2774                 dec->reserve = 0;
2775                 dec->mvc_support = 1;
2776
2777                 if (!of_machine_is_compatible("rockchip,rk3036")) {
2778                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2779
2780                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2781                         enc->h264_enabled = 1;
2782                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2783                         enc->jpeg_enabled = 1;
2784                         enc->vs_enabled = (config_reg >> 24) & 1;
2785                         enc->rgb_enabled = (config_reg >> 28) & 1;
2786                         enc->reg_size = data->reg_size;
2787                         enc->reserv[0] = 0;
2788                         enc->reserv[1] = 0;
2789                 }
2790
2791                 pservice->auto_freq = true;
2792                 vpu_debug(DEBUG_EXTRA_INFO,
2793                           "vpu_service set to auto frequency mode\n");
2794                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2795
2796                 pservice->bug_dec_addr = of_machine_is_compatible
2797                         ("rockchip,rk30xx");
2798         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2799                 pservice->auto_freq = true;
2800                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2801         } else {
2802                 /* disable frequency switch in hevc.*/
2803                 pservice->auto_freq = false;
2804         }
2805 }
2806
2807 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2808 {
2809         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2810                   task->name, irq_status, task->error_mask);
2811
2812         return (task->error_mask & irq_status) ? true : false;
2813 }
2814
2815 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2816 {
2817         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2818         struct vpu_service_info *pservice = data->pservice;
2819         struct vpu_task_info *task = NULL;
2820         struct vpu_device *dev = &data->dec_dev;
2821         u32 hw_id = data->hw_info->hw_id;
2822         u32 raw_status;
2823         u32 dec_status;
2824
2825         task = &data->task_info[TASK_DEC];
2826
2827         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2828         dec_status = raw_status;
2829
2830         vpu_debug(DEBUG_TASK_INFO,
2831                   "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2832                   task->reg_irq, dec_status,
2833                   task->irq_mask, task->ready_mask, task->error_mask);
2834
2835         if (dec_status & task->irq_mask) {
2836                 time_record(task, 1);
2837                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2838                           dec_status);
2839                 if ((dec_status & 0x40001) == 0x40001) {
2840                         do {
2841                                 dec_status =
2842                                         readl_relaxed(dev->regs +
2843                                                 task->reg_irq);
2844                         } while ((dec_status & 0x40001) == 0x40001);
2845                 }
2846
2847                 if (check_irq_err(task, dec_status))
2848                         atomic_add(1, &pservice->reset_request);
2849
2850                 writel_relaxed(0, dev->regs + task->reg_irq);
2851
2852                 /* set clock gating to save power */
2853                 writel(task->gating_mask, dev->regs + task->reg_en);
2854
2855                 atomic_add(1, &dev->irq_count_codec);
2856                 time_diff(task);
2857         }
2858
2859         task = &data->task_info[TASK_PP];
2860         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2861                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2862
2863                 if (pp_status & task->irq_mask) {
2864                         time_record(task, 1);
2865                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2866                                   pp_status);
2867
2868                         if (check_irq_err(task, dec_status))
2869                                 atomic_add(1, &pservice->reset_request);
2870
2871                         /* clear pp IRQ */
2872                         writel_relaxed(pp_status & (~task->reg_irq),
2873                                        dev->regs + task->irq_mask);
2874                         atomic_add(1, &dev->irq_count_pp);
2875                         time_diff(task);
2876                 }
2877         }
2878
2879         pservice->irq_status = raw_status;
2880
2881         if (atomic_read(&dev->irq_count_pp) ||
2882             atomic_read(&dev->irq_count_codec))
2883                 return IRQ_WAKE_THREAD;
2884         else
2885                 return IRQ_NONE;
2886 }
2887
2888 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2889 {
2890         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2891         struct vpu_service_info *pservice = data->pservice;
2892         struct vpu_device *dev = &data->dec_dev;
2893
2894         mutex_lock(&pservice->lock);
2895         if (atomic_read(&dev->irq_count_codec)) {
2896                 atomic_sub(1, &dev->irq_count_codec);
2897                 if (pservice->reg_codec == NULL) {
2898                         vpu_err("error: dec isr with no task waiting\n");
2899                 } else {
2900                         reg_from_run_to_done(data, pservice->reg_codec);
2901                         /* avoid vpu timeout and can't recover problem */
2902                         if (data->mode == VCODEC_RUNNING_MODE_VPU)
2903                                 VDPU_SOFT_RESET(data->regs);
2904                 }
2905         }
2906
2907         if (atomic_read(&dev->irq_count_pp)) {
2908                 atomic_sub(1, &dev->irq_count_pp);
2909                 if (pservice->reg_pproc == NULL)
2910                         vpu_err("error: pp isr with no task waiting\n");
2911                 else
2912                         reg_from_run_to_done(data, pservice->reg_pproc);
2913         }
2914
2915         queue_work(pservice->set_workq, &data->set_work);
2916         mutex_unlock(&pservice->lock);
2917         return IRQ_HANDLED;
2918 }
2919
2920 static irqreturn_t vepu_irq(int irq, void *dev_id)
2921 {
2922         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2923         struct vpu_service_info *pservice = data->pservice;
2924         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2925         struct vpu_device *dev = &data->enc_dev;
2926         u32 irq_status;
2927
2928         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2929
2930         vpu_debug(DEBUG_TASK_INFO,
2931                   "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2932                   task->reg_irq, irq_status,
2933                   task->irq_mask, task->ready_mask, task->error_mask);
2934
2935         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2936
2937         if (likely(irq_status & task->irq_mask)) {
2938                 time_record(task, 1);
2939
2940                 if (check_irq_err(task, irq_status))
2941                         atomic_add(1, &pservice->reset_request);
2942
2943                 /* clear enc IRQ */
2944                 writel_relaxed(irq_status & (~task->irq_mask),
2945                                dev->regs + task->reg_irq);
2946
2947                 atomic_add(1, &dev->irq_count_codec);
2948                 time_diff(task);
2949         }
2950
2951         pservice->irq_status = irq_status;
2952
2953         if (atomic_read(&dev->irq_count_codec))
2954                 return IRQ_WAKE_THREAD;
2955         else
2956                 return IRQ_NONE;
2957 }
2958
2959 static irqreturn_t vepu_isr(int irq, void *dev_id)
2960 {
2961         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2962         struct vpu_service_info *pservice = data->pservice;
2963         struct vpu_device *dev = &data->enc_dev;
2964
2965         mutex_lock(&pservice->lock);
2966         if (atomic_read(&dev->irq_count_codec)) {
2967                 atomic_sub(1, &dev->irq_count_codec);
2968                 if (NULL == pservice->reg_codec)
2969                         vpu_err("error: enc isr with no task waiting\n");
2970                 else
2971                         reg_from_run_to_done(data, pservice->reg_codec);
2972         }
2973         queue_work(pservice->set_workq, &data->set_work);
2974         mutex_unlock(&pservice->lock);
2975
2976         return IRQ_HANDLED;
2977 }
2978
2979 module_platform_driver(vcodec_driver);
2980 MODULE_LICENSE("GPL v2");