drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming, chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *         Jung Zhao, jung.zhao@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/clk.h>
  21 #include <linux/compat.h>
  22 #include <linux/delay.h>
  23 #include <linux/init.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/mm.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/reset.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/wakelock.h>
  33 #include <linux/cdev.h>
  34 #include <linux/of.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/regmap.h>
  38 #include <linux/mfd/syscon.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/iopoll.h>
  43
  44 #include <linux/rockchip/cru.h>
  45 #include <linux/rockchip/pmu.h>
  46 #include <linux/rockchip/grf.h>
  47
  48 #include <linux/dma-buf.h>
  49 #include <linux/rockchip-iovmm.h>
  50
  51 #include "vcodec_hw_info.h"
  52 #include "vcodec_hw_vpu.h"
  53 #include "vcodec_hw_rkv.h"
  54 #include "vcodec_hw_vpu2.h"
  55
  56 #include "vcodec_service.h"
  57
  58 #include "vcodec_iommu_ops.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 #define MHZ                                     (1000 * 1000)
  90 #define SIZE_REG(reg)                           ((reg) * 4)
  91
  92 #define VCODEC_CLOCK_ENABLE     1
  93 #define EXTRA_INFO_MAGIC        0x4C4A46
  94
  95 static int debug;
  96 module_param(debug, int, S_IRUGO | S_IWUSR);
  97 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  98 /*
  99  * hardware information organization
 100  *
 101  * In order to support multiple hardware with different version the hardware
 102  * information is organized as follow:
 103  *
 104  * 1. First, index hardware by register size / position.
 105  *    These information is fix for each hardware and do not relate to runtime
 106  *    work flow. It only related to resource allocation.
 107  *    Descriptor: struct vpu_hw_info
 108  *
 109  * 2. Then, index hardware by runtime configuration
 110  *    These information is related to runtime setting behave including enable
 111  *    register, irq register and other key control flag
 112  *    Descriptor: struct vpu_task_info
 113  *
 114  * 3. Final, on iommu case the fd translation is required
 115  *    Descriptor: struct vpu_trans_info
 116  */
 117
 118 enum VPU_FREQ {
 119         VPU_FREQ_200M,
 120         VPU_FREQ_266M,
 121         VPU_FREQ_300M,
 122         VPU_FREQ_400M,
 123         VPU_FREQ_500M,
 124         VPU_FREQ_600M,
 125         VPU_FREQ_DEFAULT,
 126         VPU_FREQ_BUT,
 127 };
 128
 129 struct extra_info_elem {
 130         u32 index;
 131         u32 offset;
 132 };
 133
 134
 135 struct extra_info_for_iommu {
 136         u32 magic;
 137         u32 cnt;
 138         struct extra_info_elem elem[20];
 139 };
 140
 141 static const struct vcodec_info vcodec_info_set[] = {
 142         {
 143                 .hw_id          = VPU_ID_8270,
 144                 .hw_info        = &hw_vpu_8270,
 145                 .task_info      = task_vpu,
 146                 .trans_info     = trans_vpu,
 147         },
 148         {
 149                 .hw_id          = VPU_ID_4831,
 150                 .hw_info        = &hw_vpu_4831,
 151                 .task_info      = task_vpu,
 152                 .trans_info     = trans_vpu,
 153         },
 154         {
 155                 .hw_id          = VPU_DEC_ID_9190,
 156                 .hw_info        = &hw_vpu_9190,
 157                 .task_info      = task_vpu,
 158                 .trans_info     = trans_vpu,
 159         },
 160         {
 161                 .hw_id          = HEVC_ID,
 162                 .hw_info        = &hw_rkhevc,
 163                 .task_info      = task_rkv,
 164                 .trans_info     = trans_rkv,
 165         },
 166         {
 167                 .hw_id          = RKV_DEC_ID,
 168                 .hw_info        = &hw_rkvdec,
 169                 .task_info      = task_rkv,
 170                 .trans_info     = trans_rkv,
 171         },
 172         {
 173                 .hw_id          = VPU2_ID,
 174                 .hw_info        = &hw_vpu2,
 175                 .task_info      = task_vpu2,
 176                 .trans_info     = trans_vpu2,
 177         },
 178 };
 179
 180 /* Both VPU1 and VPU2 */
 181 static const struct vcodec_device_info vpu_device_info = {
 182         .device_type = VCODEC_DEVICE_TYPE_VPUX,
 183         .name = "vpu-service",
 184 };
 185
 186 static const struct vcodec_device_info vpu_combo_device_info = {
 187         .device_type = VCODEC_DEVICE_TYPE_VPUC,
 188         .name = "vpu-combo",
 189 };
 190
 191 static const struct vcodec_device_info hevc_device_info = {
 192         .device_type = VCODEC_DEVICE_TYPE_HEVC,
 193         .name = "hevc-service",
 194 };
 195
 196 static const struct vcodec_device_info rkvd_device_info = {
 197         .device_type = VCODEC_DEVICE_TYPE_RKVD,
 198         .name = "rkvdec",
 199 };
 200
 201 #define DEBUG
 202 #ifdef DEBUG
 203 #define vpu_debug_func(type, fmt, args...)                      \
 204         do {                                                    \
 205                 if (unlikely(debug & type)) {                   \
 206                         pr_info("%s:%d: " fmt,                  \
 207                                  __func__, __LINE__, ##args);   \
 208                 }                                               \
 209         } while (0)
 210 #define vpu_debug(type, fmt, args...)                           \
 211         do {                                                    \
 212                 if (unlikely(debug & type)) {                   \
 213                         pr_info(fmt, ##args);                   \
 214                 }                                               \
 215         } while (0)
 216 #else
 217 #define vpu_debug_func(level, fmt, args...)
 218 #define vpu_debug(level, fmt, args...)
 219 #endif
 220
 221 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 222 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 223
 224 #define vpu_err(fmt, args...)                           \
 225                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 226
 227 enum VPU_DEC_FMT {
 228         VPU_DEC_FMT_H264,
 229         VPU_DEC_FMT_MPEG4,
 230         VPU_DEC_FMT_H263,
 231         VPU_DEC_FMT_JPEG,
 232         VPU_DEC_FMT_VC1,
 233         VPU_DEC_FMT_MPEG2,
 234         VPU_DEC_FMT_MPEG1,
 235         VPU_DEC_FMT_VP6,
 236         VPU_DEC_FMT_RESERV0,
 237         VPU_DEC_FMT_VP7,
 238         VPU_DEC_FMT_VP8,
 239         VPU_DEC_FMT_AVS,
 240         VPU_DEC_FMT_RES
 241 };
 242
 243 /**
 244  * struct for process session which connect to vpu
 245  *
 246  * @author ChenHengming (2011-5-3)
 247  */
 248 struct vpu_session {
 249         enum VPU_CLIENT_TYPE type;
 250         /* a linked list of data so we can access them for debugging */
 251         struct list_head list_session;
 252         /* a linked list of register data waiting for process */
 253         struct list_head waiting;
 254         /* a linked list of register data in processing */
 255         struct list_head running;
 256         /* a linked list of register data processed */
 257         struct list_head done;
 258         wait_queue_head_t wait;
 259         pid_t pid;
 260         atomic_t task_running;
 261 };
 262
 263 /**
 264  * struct for process register set
 265  *
 266  * @author ChenHengming (2011-5-4)
 267  */
 268 struct vpu_reg {
 269         enum VPU_CLIENT_TYPE type;
 270         enum VPU_FREQ freq;
 271         struct vpu_session *session;
 272         struct vpu_subdev_data *data;
 273         struct vpu_task_info *task;
 274         const struct vpu_trans_info *trans;
 275
 276         /* link to vpu service session */
 277         struct list_head session_link;
 278         /* link to register set list */
 279         struct list_head status_link;
 280
 281         unsigned long size;
 282         struct list_head mem_region_list;
 283         u32 dec_base;
 284         u32 *reg;
 285 };
 286
 287 struct vpu_device {
 288         atomic_t irq_count_codec;
 289         atomic_t irq_count_pp;
 290         unsigned int iosize;
 291         u32 *regs;
 292 };
 293
 294 enum vcodec_device_id {
 295         VCODEC_DEVICE_ID_VPU,
 296         VCODEC_DEVICE_ID_HEVC,
 297         VCODEC_DEVICE_ID_COMBO,
 298         VCODEC_DEVICE_ID_RKVDEC,
 299         VCODEC_DEVICE_ID_BUTT
 300 };
 301
 302 enum VCODEC_RUNNING_MODE {
 303         VCODEC_RUNNING_MODE_NONE = -1,
 304         VCODEC_RUNNING_MODE_VPU,
 305         VCODEC_RUNNING_MODE_HEVC,
 306         VCODEC_RUNNING_MODE_RKVDEC
 307 };
 308
 309 struct vcodec_mem_region {
 310         struct list_head srv_lnk;
 311         struct list_head reg_lnk;
 312         struct list_head session_lnk;
 313         unsigned long iova;     /* virtual address for iommu */
 314         unsigned long len;
 315         u32 reg_idx;
 316         int hdl;
 317 };
 318
 319 enum vpu_ctx_state {
 320         MMU_ACTIVATED   = BIT(0)
 321 };
 322
 323 struct vpu_subdev_data {
 324         struct cdev cdev;
 325         dev_t dev_t;
 326         struct class *cls;
 327         struct device *child_dev;
 328
 329         int irq_enc;
 330         int irq_dec;
 331         struct vpu_service_info *pservice;
 332
 333         u32 *regs;
 334         enum VCODEC_RUNNING_MODE mode;
 335         struct list_head lnk_service;
 336
 337         struct device *dev;
 338
 339         struct vpu_device enc_dev;
 340         struct vpu_device dec_dev;
 341
 342         enum VPU_HW_ID hw_id;
 343         struct vpu_hw_info *hw_info;
 344         struct vpu_task_info *task_info;
 345         const struct vpu_trans_info *trans_info;
 346
 347         u32 reg_size;
 348         unsigned long state;
 349
 350 #ifdef CONFIG_DEBUG_FS
 351         struct dentry *debugfs_dir;
 352         struct dentry *debugfs_file_regs;
 353 #endif
 354
 355         struct device *mmu_dev;
 356         struct vcodec_iommu_info *iommu_info;
 357         struct work_struct set_work;
 358 };
 359
 360 struct vpu_service_info {
 361         struct wake_lock wake_lock;
 362         struct delayed_work power_off_work;
 363         struct wake_lock set_wake_lock;
 364         struct workqueue_struct *set_workq;
 365         ktime_t last; /* record previous power-on time */
 366         /* vpu service structure global lock */
 367         struct mutex lock;
 368         /* link to link_reg in struct vpu_reg */
 369         struct list_head waiting;
 370         /* link to link_reg in struct vpu_reg */
 371         struct list_head running;
 372         /* link to link_reg in struct vpu_reg */
 373         struct list_head done;
 374         /* link to list_session in struct vpu_session */
 375         struct list_head session;
 376         atomic_t total_running;
 377         atomic_t enabled;
 378         atomic_t power_on_cnt;
 379         atomic_t power_off_cnt;
 380         atomic_t service_on;
 381         struct mutex shutdown_lock;
 382         struct vpu_reg *reg_codec;
 383         struct vpu_reg *reg_pproc;
 384         struct vpu_reg *reg_resev;
 385         struct vpu_dec_config dec_config;
 386         struct vpu_enc_config enc_config;
 387
 388         bool auto_freq;
 389         bool bug_dec_addr;
 390         atomic_t freq_status;
 391
 392         struct clk *aclk_vcodec;
 393         struct clk *hclk_vcodec;
 394         struct clk *clk_core;
 395         struct clk *clk_cabac;
 396         struct clk *pd_video;
 397
 398 #ifdef CONFIG_RESET_CONTROLLER
 399         struct reset_control *rst_a;
 400         struct reset_control *rst_h;
 401         struct reset_control *rst_v;
 402 #endif
 403         struct device *dev;
 404
 405         u32 irq_status;
 406         atomic_t reset_request;
 407         struct list_head mem_region_list;
 408
 409         enum vcodec_device_id dev_id;
 410
 411         enum VCODEC_RUNNING_MODE curr_mode;
 412         u32 prev_mode;
 413
 414         struct delayed_work simulate_work;
 415
 416         u32 mode_bit;
 417         u32 mode_ctrl;
 418         u32 *reg_base;
 419         u32 ioaddr;
 420         struct regmap *grf;
 421         u32 *grf_base;
 422
 423         char *name;
 424
 425         u32 subcnt;
 426         struct list_head subdev_list;
 427
 428         u32 alloc_type;
 429 };
 430
 431 struct vpu_request {
 432         u32 *req;
 433         u32 size;
 434 };
 435
 436 #ifdef CONFIG_COMPAT
 437 struct compat_vpu_request {
 438         compat_uptr_t req;
 439         u32 size;
 440 };
 441 #endif
 442
 443 #define VDPU_SOFT_RESET_REG     101
 444 #define VDPU_CLEAN_CACHE_REG    516
 445 #define VEPU_CLEAN_CACHE_REG    772
 446 #define HEVC_CLEAN_CACHE_REG    260
 447
 448 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 449
 450 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 451 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 452 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 453 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 454
 455 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 456 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 457
 458 static void *vcodec_get_drv_data(struct platform_device *pdev);
 459
 460 static void vpu_service_power_on(struct vpu_subdev_data *data,
 461                                  struct vpu_service_info *pservice);
 462
 463 static void time_record(struct vpu_task_info *task, int is_end)
 464 {
 465         if (unlikely(debug & DEBUG_TIMING) && task)
 466                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 467 }
 468
 469 static void time_diff(struct vpu_task_info *task)
 470 {
 471         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 472                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 473                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 474 }
 475
 476 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 477 {
 478         int bits;
 479         u32 raw = 0;
 480         struct vpu_service_info *pservice = data->pservice;
 481         struct vpu_subdev_data *subdata, *n;
 482
 483         if (pservice->subcnt < 2) {
 484                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 485                         set_bit(MMU_ACTIVATED, &data->state);
 486
 487                         if (atomic_read(&pservice->enabled)) {
 488                                 if (vcodec_iommu_attach(data->iommu_info))
 489                                         dev_err(data->dev,
 490                                                 "vcodec service attach failed\n"
 491                                                 );
 492                                 else
 493                                         BUG_ON(
 494                                                !atomic_read(&pservice->enabled)
 495                                                );
 496                         }
 497                 }
 498                 return;
 499         }
 500
 501         if (pservice->curr_mode == data->mode)
 502                 return;
 503
 504         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 505         list_for_each_entry_safe(subdata, n,
 506                                  &pservice->subdev_list, lnk_service) {
 507                 if (data != subdata && subdata->mmu_dev &&
 508                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 509                         clear_bit(MMU_ACTIVATED, &subdata->state);
 510                         vcodec_iommu_detach(subdata->iommu_info);
 511                 }
 512         }
 513         bits = 1 << pservice->mode_bit;
 514 #ifdef CONFIG_MFD_SYSCON
 515         if (pservice->grf) {
 516                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 517
 518                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 519                         regmap_write(pservice->grf, pservice->mode_ctrl,
 520                                      raw | bits | (bits << 16));
 521                 else
 522                         regmap_write(pservice->grf, pservice->mode_ctrl,
 523                                      (raw & (~bits)) | (bits << 16));
 524         } else if (pservice->grf_base) {
 525                 u32 *grf_base = pservice->grf_base;
 526
 527                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 528                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 529                         writel_relaxed(raw | bits | (bits << 16),
 530                                        grf_base + pservice->mode_ctrl / 4);
 531                 else
 532                         writel_relaxed((raw & (~bits)) | (bits << 16),
 533                                        grf_base + pservice->mode_ctrl / 4);
 534         } else {
 535                 vpu_err("no grf resource define, switch decoder failed\n");
 536                 return;
 537         }
 538 #else
 539         if (pservice->grf_base) {
 540                 u32 *grf_base = pservice->grf_base;
 541
 542                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 543                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 544                         writel_relaxed(raw | bits | (bits << 16),
 545                                        grf_base + pservice->mode_ctrl / 4);
 546                 else
 547                         writel_relaxed((raw & (~bits)) | (bits << 16),
 548                                        grf_base + pservice->mode_ctrl / 4);
 549         } else {
 550                 vpu_err("no grf resource define, switch decoder failed\n");
 551                 return;
 552         }
 553 #endif
 554         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 555                 set_bit(MMU_ACTIVATED, &data->state);
 556                 if (atomic_read(&pservice->enabled))
 557                         vcodec_iommu_attach(data->iommu_info);
 558                 else
 559                         /* FIXME BUG_ON should not be used in mass produce */
 560                         BUG_ON(!atomic_read(&pservice->enabled));
 561         }
 562
 563         pservice->prev_mode = pservice->curr_mode;
 564         pservice->curr_mode = data->mode;
 565 }
 566
 567 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 568 {
 569         /*
 570          * In case of VPU Combo, it require HW switch its running mode
 571          * before the other HW component start work. set current HW running
 572          * mode to none, can ensure HW switch to its reqired mode properly.
 573          */
 574         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 575 }
 576
 577 static int vpu_get_clk(struct vpu_service_info *pservice)
 578 {
 579 #if VCODEC_CLOCK_ENABLE
 580         struct device *dev = pservice->dev;
 581
 582         switch (pservice->dev_id) {
 583         case VCODEC_DEVICE_ID_HEVC:
 584                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 585                 if (IS_ERR(pservice->pd_video)) {
 586                         pservice->pd_video = NULL;
 587                         dev_info(dev, "failed on clk_get pd_hevc\n");
 588                 }
 589         case VCODEC_DEVICE_ID_COMBO:
 590         case VCODEC_DEVICE_ID_RKVDEC:
 591                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 592                 if (IS_ERR(pservice->clk_cabac)) {
 593                         dev_err(dev, "failed on clk_get clk_cabac\n");
 594                         pservice->clk_cabac = NULL;
 595                 }
 596                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 597                 if (IS_ERR(pservice->clk_core)) {
 598                         dev_err(dev, "failed on clk_get clk_core\n");
 599                         pservice->clk_core = NULL;
 600                         return -1;
 601                 }
 602         case VCODEC_DEVICE_ID_VPU:
 603                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 604                 if (IS_ERR(pservice->aclk_vcodec)) {
 605                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 606                         pservice->aclk_vcodec = NULL;
 607                         return -1;
 608                 }
 609
 610                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 611                 if (IS_ERR(pservice->hclk_vcodec)) {
 612                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 613                         pservice->hclk_vcodec = NULL;
 614                         return -1;
 615                 }
 616                 if (pservice->pd_video == NULL) {
 617                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 618                         if (IS_ERR(pservice->pd_video)) {
 619                                 pservice->pd_video = NULL;
 620                                 dev_info(dev, "do not have pd_video\n");
 621                         }
 622                 }
 623                 break;
 624         default:
 625                 break;
 626         }
 627
 628         return 0;
 629 #else
 630         return 0;
 631 #endif
 632 }
 633
 634 static void _vpu_reset(struct vpu_subdev_data *data)
 635 {
 636         struct vpu_service_info *pservice = data->pservice;
 637         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 638
 639         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 640                 type = IDLE_REQ_HEVC;
 641
 642         dev_info(pservice->dev, "resetting...\n");
 643         WARN_ON(pservice->reg_codec != NULL);
 644         WARN_ON(pservice->reg_pproc != NULL);
 645         WARN_ON(pservice->reg_resev != NULL);
 646         pservice->reg_codec = NULL;
 647         pservice->reg_pproc = NULL;
 648         pservice->reg_resev = NULL;
 649
 650 #ifdef CONFIG_RESET_CONTROLLER
 651         dev_info(pservice->dev, "for 3288/3368...");
 652         if (of_machine_is_compatible("rockchip,rk3288"))
 653                 rockchip_pmu_idle_request(pservice->dev, true);
 654         if (pservice->rst_a && pservice->rst_h) {
 655                 dev_info(pservice->dev, "vpu reset in\n");
 656
 657                 if (pservice->rst_v)
 658                         reset_control_assert(pservice->rst_v);
 659                 reset_control_assert(pservice->rst_a);
 660                 reset_control_assert(pservice->rst_h);
 661                 udelay(5);
 662
 663                 reset_control_deassert(pservice->rst_h);
 664                 reset_control_deassert(pservice->rst_a);
 665                 if (pservice->rst_v)
 666                         reset_control_deassert(pservice->rst_v);
 667         } else if (pservice->rst_v) {
 668                 dev_info(pservice->dev, "hevc reset in\n");
 669                 reset_control_assert(pservice->rst_v);
 670                 udelay(5);
 671
 672                 reset_control_deassert(pservice->rst_v);
 673         }
 674         if (of_machine_is_compatible("rockchip,rk3288"))
 675                 rockchip_pmu_idle_request(pservice->dev, false);
 676 #endif
 677 }
 678
 679 static void vpu_reset(struct vpu_subdev_data *data)
 680 {
 681         struct vpu_service_info *pservice = data->pservice;
 682
 683         _vpu_reset(data);
 684         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 685                 clear_bit(MMU_ACTIVATED, &data->state);
 686                 if (atomic_read(&pservice->enabled)) {
 687                         /* Need to reset iommu */
 688                         vcodec_iommu_detach(data->iommu_info);
 689                 } else {
 690                         /* FIXME BUG_ON should not be used in mass produce */
 691                         BUG_ON(!atomic_read(&pservice->enabled));
 692                 }
 693         }
 694
 695         atomic_set(&pservice->reset_request, 0);
 696         dev_info(pservice->dev, "reset done\n");
 697 }
 698
 699 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 700 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 701                                       struct vpu_session *session)
 702 {
 703         struct vpu_reg *reg, *n;
 704
 705         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 706                 reg_deinit(data, reg);
 707         }
 708         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 709                 reg_deinit(data, reg);
 710         }
 711         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 712                 reg_deinit(data, reg);
 713         }
 714 }
 715
 716 static void vpu_service_clear(struct vpu_subdev_data *data)
 717 {
 718         struct vpu_reg *reg, *n;
 719         struct vpu_session *session, *s;
 720         struct vpu_service_info *pservice = data->pservice;
 721
 722         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 723                 reg_deinit(reg->data, reg);
 724         }
 725
 726         /* wake up session wait event to prevent the timeout hw reset
 727          * during reboot procedure.
 728          */
 729         list_for_each_entry_safe(session, s,
 730                                  &pservice->session, list_session)
 731                 wake_up(&session->wait);
 732 }
 733
 734 static void vpu_service_dump(struct vpu_service_info *pservice)
 735 {
 736 }
 737
 738
 739 static void vpu_service_power_off(struct vpu_service_info *pservice)
 740 {
 741         int total_running;
 742         struct vpu_subdev_data *data = NULL, *n;
 743         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 744
 745         if (!ret)
 746                 return;
 747
 748         total_running = atomic_read(&pservice->total_running);
 749         if (total_running) {
 750                 pr_alert("alert: power off when %d task running!!\n",
 751                          total_running);
 752                 mdelay(50);
 753                 pr_alert("alert: delay 50 ms for running task\n");
 754                 vpu_service_dump(pservice);
 755         }
 756
 757         dev_dbg(pservice->dev, "power off...\n");
 758
 759         udelay(5);
 760
 761         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 762                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 763                         clear_bit(MMU_ACTIVATED, &data->state);
 764                         vcodec_iommu_detach(data->iommu_info);
 765                 }
 766         }
 767         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 768         pm_runtime_put(pservice->dev);
 769 #if VCODEC_CLOCK_ENABLE
 770                 if (pservice->pd_video)
 771                         clk_disable_unprepare(pservice->pd_video);
 772                 if (pservice->hclk_vcodec)
 773                         clk_disable_unprepare(pservice->hclk_vcodec);
 774                 if (pservice->aclk_vcodec)
 775                         clk_disable_unprepare(pservice->aclk_vcodec);
 776                 if (pservice->clk_core)
 777                         clk_disable_unprepare(pservice->clk_core);
 778                 if (pservice->clk_cabac)
 779                         clk_disable_unprepare(pservice->clk_cabac);
 780 #endif
 781
 782         atomic_add(1, &pservice->power_off_cnt);
 783         wake_unlock(&pservice->wake_lock);
 784         dev_dbg(pservice->dev, "power off done\n");
 785 }
 786
 787 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 788 {
 789         queue_delayed_work(system_wq, &pservice->power_off_work,
 790                            VPU_POWER_OFF_DELAY);
 791 }
 792
 793 static void vpu_power_off_work(struct work_struct *work_s)
 794 {
 795         struct delayed_work *dlwork = container_of(work_s,
 796                         struct delayed_work, work);
 797         struct vpu_service_info *pservice = container_of(dlwork,
 798                         struct vpu_service_info, power_off_work);
 799
 800         if (mutex_trylock(&pservice->lock)) {
 801                 vpu_service_power_off(pservice);
 802                 mutex_unlock(&pservice->lock);
 803         } else {
 804                 /* Come back later if the device is busy... */
 805                 vpu_queue_power_off_work(pservice);
 806         }
 807 }
 808
 809 static void vpu_service_power_on(struct vpu_subdev_data *data,
 810                                  struct vpu_service_info *pservice)
 811 {
 812         int ret;
 813         ktime_t now = ktime_get();
 814
 815         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC ||
 816             atomic_read(&pservice->power_on_cnt)) {
 817                 /* NSEC_PER_SEC */
 818                 cancel_delayed_work_sync(&pservice->power_off_work);
 819                 vpu_queue_power_off_work(pservice);
 820                 pservice->last = now;
 821         }
 822         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 823         if (!ret)
 824                 return;
 825
 826         dev_dbg(pservice->dev, "power on\n");
 827
 828 #define BIT_VCODEC_CLK_SEL      (1<<10)
 829         if (of_machine_is_compatible("rockchip,rk3126"))
 830                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 831                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 832                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 833
 834 #if VCODEC_CLOCK_ENABLE
 835         if (pservice->aclk_vcodec)
 836                 clk_prepare_enable(pservice->aclk_vcodec);
 837         if (pservice->hclk_vcodec)
 838                 clk_prepare_enable(pservice->hclk_vcodec);
 839         if (pservice->clk_core)
 840                 clk_prepare_enable(pservice->clk_core);
 841         if (pservice->clk_cabac)
 842                 clk_prepare_enable(pservice->clk_cabac);
 843         if (pservice->pd_video)
 844                 clk_prepare_enable(pservice->pd_video);
 845 #endif
 846         pm_runtime_get_sync(pservice->dev);
 847
 848         udelay(5);
 849         atomic_add(1, &pservice->power_on_cnt);
 850         wake_lock(&pservice->wake_lock);
 851 }
 852
 853 static inline bool reg_check_interlace(struct vpu_reg *reg)
 854 {
 855         u32 type = (reg->reg[3] & (1 << 23));
 856
 857         return (type > 0);
 858 }
 859
 860 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 861 {
 862         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 863
 864         return type;
 865 }
 866
 867 static inline int reg_probe_width(struct vpu_reg *reg)
 868 {
 869         int width_in_mb = reg->reg[4] >> 23;
 870
 871         return width_in_mb * 16;
 872 }
 873
 874 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 875 {
 876         int y_virstride = reg->reg[8];
 877
 878         return y_virstride;
 879 }
 880
 881 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 882                 struct vpu_session *session,
 883                 struct vpu_reg *reg,
 884                 int fd)
 885 {
 886         int hdl;
 887         int ret = 0;
 888         struct vcodec_mem_region *mem_region;
 889
 890         hdl = vcodec_iommu_import(data->iommu_info, session, fd);
 891         if (hdl < 0)
 892                 return hdl;
 893
 894         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 895         if (mem_region == NULL) {
 896                 vpu_err("allocate memory for iommu memory region failed\n");
 897                 vcodec_iommu_free(data->iommu_info, session, hdl);
 898                 return -ENOMEM;
 899         }
 900
 901         mem_region->hdl = hdl;
 902         ret = vcodec_iommu_map_iommu(data->iommu_info, session, mem_region->hdl,
 903                                      &mem_region->iova, &mem_region->len);
 904         if (ret < 0) {
 905                 vpu_err("fd %d ion map iommu failed\n", fd);
 906                 kfree(mem_region);
 907                 vcodec_iommu_free(data->iommu_info, session, hdl);
 908
 909                 return -EFAULT;
 910         }
 911         INIT_LIST_HEAD(&mem_region->reg_lnk);
 912         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 913         return mem_region->iova;
 914 }
 915
 916 /*
 917  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 918  * it by pps id in video stream data.
 919  *
 920  * So we need to translate the address in iommu case. The address data is also
 921  * 10bit fd + 22bit offset mode.
 922  * Because userspace decoder do not give the pps id in the register file sets
 923  * kernel driver need to translate each scaling list address in pps buffer which
 924  * means 256 pps for H.264, 64 pps for H.265.
 925  *
 926  * In order to optimize the performance kernel driver ask userspace decoder to
 927  * set all scaling list address in pps buffer to the same one which will be used
 928  * on current decoding task. Then kernel driver can only translate the first
 929  * address then copy it all pps buffer.
 930  */
 931 static int fill_scaling_list_addr_in_pps(
 932                 struct vpu_subdev_data *data,
 933                 struct vpu_reg *reg,
 934                 char *pps,
 935                 int pps_info_count,
 936                 int pps_info_size,
 937                 int scaling_list_addr_offset)
 938 {
 939         int base = scaling_list_addr_offset;
 940         int scaling_fd = 0;
 941         u32 scaling_offset;
 942
 943         scaling_offset  = (u32)pps[base + 0];
 944         scaling_offset += (u32)pps[base + 1] << 8;
 945         scaling_offset += (u32)pps[base + 2] << 16;
 946         scaling_offset += (u32)pps[base + 3] << 24;
 947
 948         scaling_fd = scaling_offset & 0x3ff;
 949         scaling_offset = scaling_offset >> 10;
 950
 951         if (scaling_fd > 0) {
 952                 int i = 0;
 953                 u32 tmp = vcodec_fd_to_iova(data, reg->session, reg,
 954                                             scaling_fd);
 955
 956                 if (IS_ERR_VALUE(tmp))
 957                         return -1;
 958                 tmp += scaling_offset;
 959
 960                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 961                         pps[base + 0] = (tmp >>  0) & 0xff;
 962                         pps[base + 1] = (tmp >>  8) & 0xff;
 963                         pps[base + 2] = (tmp >> 16) & 0xff;
 964                         pps[base + 3] = (tmp >> 24) & 0xff;
 965                 }
 966         }
 967
 968         return 0;
 969 }
 970
 971 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data,
 972                                 struct vpu_session *session,
 973                                 const u8 *tbl,
 974                                 int size, struct vpu_reg *reg,
 975                                 struct extra_info_for_iommu *ext_inf)
 976 {
 977         struct vpu_service_info *pservice = data->pservice;
 978         struct vpu_task_info *task = reg->task;
 979         enum FORMAT_TYPE type;
 980         int hdl;
 981         int ret = 0;
 982         struct vcodec_mem_region *mem_region;
 983         int i;
 984         int offset = 0;
 985
 986         if (tbl == NULL || size <= 0) {
 987                 dev_err(pservice->dev, "input arguments invalidate\n");
 988                 return -EINVAL;
 989         }
 990
 991         if (task->get_fmt)
 992                 type = task->get_fmt(reg->reg);
 993         else {
 994                 dev_err(pservice->dev, "invalid task with NULL get_fmt\n");
 995                 return -EINVAL;
 996         }
 997
 998         for (i = 0; i < size; i++) {
 999                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
1000
1001                 /* if userspace do not set the fd at this register, skip */
1002                 if (usr_fd == 0)
1003                         continue;
1004
1005                 /*
1006                  * for avoiding cache sync issue, we need to map/unmap
1007                  * input buffer every time. FIX ME, if it is unnecessary
1008                  */
1009                 if (task->reg_rlc == tbl[i])
1010                         vcodec_iommu_free_fd(data->iommu_info, session, usr_fd);
1011                 /*
1012                  * special offset scale case
1013                  *
1014                  * This translation is for fd + offset translation.
1015                  * One register has 32bits. We need to transfer both buffer file
1016                  * handle and the start address offset so we packet file handle
1017                  * and offset together using below format.
1018                  *
1019                  *  0~9  bit for buffer file handle range 0 ~ 1023
1020                  * 10~31 bit for offset range 0 ~ 4M
1021                  *
1022                  * But on 4K case the offset can be larger the 4M
1023                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1024                  * But MPEG4 will use the same register for colmv and it do not
1025                  * need scale.
1026                  *
1027                  * RKVdec do not have this issue.
1028                  */
1029                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1030                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1031                         offset = reg->reg[tbl[i]] >> 10 << 4;
1032                 else
1033                         offset = reg->reg[tbl[i]] >> 10;
1034
1035                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d i %d\n",
1036                           tbl[i], usr_fd, offset, i);
1037
1038                 hdl = vcodec_iommu_import(data->iommu_info, session, usr_fd);
1039
1040                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1041                         int pps_info_offset;
1042                         int pps_info_count;
1043                         int pps_info_size;
1044                         int scaling_list_addr_offset;
1045
1046                         switch (type) {
1047                         case FMT_H264D: {
1048                                 pps_info_offset = offset;
1049                                 pps_info_count = 256;
1050                                 pps_info_size = 32;
1051                                 scaling_list_addr_offset = 23;
1052                         } break;
1053                         case FMT_H265D: {
1054                                 pps_info_offset = 0;
1055                                 pps_info_count = 64;
1056                                 pps_info_size = 80;
1057                                 scaling_list_addr_offset = 74;
1058                         } break;
1059                         default: {
1060                                 pps_info_offset = 0;
1061                                 pps_info_count = 0;
1062                                 pps_info_size = 0;
1063                                 scaling_list_addr_offset = 0;
1064                         } break;
1065                         }
1066
1067                         vpu_debug(DEBUG_PPS_FILL,
1068                                   "scaling list filling parameter:\n");
1069                         vpu_debug(DEBUG_PPS_FILL,
1070                                   "pps_info_offset %d\n", pps_info_offset);
1071                         vpu_debug(DEBUG_PPS_FILL,
1072                                   "pps_info_count  %d\n", pps_info_count);
1073                         vpu_debug(DEBUG_PPS_FILL,
1074                                   "pps_info_size   %d\n", pps_info_size);
1075                         vpu_debug(DEBUG_PPS_FILL,
1076                                   "scaling_list_addr_offset %d\n",
1077                                   scaling_list_addr_offset);
1078
1079                         if (pps_info_count) {
1080                                 u8 *pps;
1081
1082                                 pps = vcodec_iommu_map_kernel
1083                                         (data->iommu_info, session, hdl);
1084
1085                                 vpu_debug(DEBUG_PPS_FILL,
1086                                           "scaling list setting pps %p\n", pps);
1087                                 pps += pps_info_offset;
1088
1089                                 fill_scaling_list_addr_in_pps
1090                                         (data, reg, pps, pps_info_count,
1091                                          pps_info_size,
1092                                          scaling_list_addr_offset);
1093
1094                                 vcodec_iommu_unmap_kernel
1095                                         (data->iommu_info, session, hdl);
1096                         }
1097                 }
1098
1099                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1100
1101                 if (!mem_region) {
1102                         vcodec_iommu_free(data->iommu_info, session, hdl);
1103                         return -ENOMEM;
1104                 }
1105
1106                 mem_region->hdl = hdl;
1107                 mem_region->reg_idx = tbl[i];
1108
1109                 ret = vcodec_iommu_map_iommu(data->iommu_info, session,
1110                                              mem_region->hdl, &mem_region->iova,
1111                                              &mem_region->len);
1112                 if (ret < 0) {
1113                         dev_err(pservice->dev,
1114                                 "reg %d fd %d ion map iommu failed\n",
1115                                 tbl[i], usr_fd);
1116                         kfree(mem_region);
1117                         vcodec_iommu_free(data->iommu_info, session, hdl);
1118                         return ret;
1119                 }
1120
1121                 /*
1122                  * special for vpu dec num 12: record decoded length
1123                  * hacking for decoded length
1124                  * NOTE: not a perfect fix, the fd is not recorded
1125                  */
1126                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1127                         reg->dec_base = mem_region->iova + offset;
1128                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1129                                   reg->dec_base);
1130                 }
1131
1132                 reg->reg[tbl[i]] = mem_region->iova + offset;
1133                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1134                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1135         }
1136
1137         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1138                 for (i = 0; i < ext_inf->cnt; i++) {
1139                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1140                                   ext_inf->elem[i].index,
1141                                   ext_inf->elem[i].offset);
1142                         reg->reg[ext_inf->elem[i].index] +=
1143                                 ext_inf->elem[i].offset;
1144                 }
1145         }
1146
1147         return 0;
1148 }
1149
1150 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1151                                         struct vpu_session *session,
1152                                         struct vpu_reg *reg,
1153                                         struct extra_info_for_iommu *ext_inf)
1154 {
1155         struct vpu_service_info *pservice = data->pservice;
1156         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1157
1158         if (type < FMT_TYPE_BUTT) {
1159                 const struct vpu_trans_info *info = &reg->trans[type];
1160                 const u8 *tbl = info->table;
1161                 int size = info->count;
1162
1163                 return vcodec_bufid_to_iova(data, session, tbl, size, reg,
1164                                             ext_inf);
1165         }
1166
1167         dev_err(pservice->dev, "found invalid format type!\n");
1168         return -EINVAL;
1169 }
1170
1171 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1172 {
1173
1174         if (!of_machine_is_compatible("rockchip,rk2928g")) {
1175                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1176                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1177                                 if (reg_probe_width(reg) > 3200) {
1178                                         /*raise frequency for 4k avc.*/
1179                                         reg->freq = VPU_FREQ_600M;
1180                                 }
1181                         } else {
1182                                 if (reg_check_interlace(reg))
1183                                         reg->freq = VPU_FREQ_400M;
1184                         }
1185                 }
1186                 if (data->hw_id == HEVC_ID) {
1187                         if (reg_probe_hevc_y_stride(reg) > 60000)
1188                                 reg->freq = VPU_FREQ_400M;
1189                 }
1190                 if (reg->type == VPU_PP)
1191                         reg->freq = VPU_FREQ_400M;
1192         }
1193 }
1194
1195 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1196                                 struct vpu_session *session,
1197                                 void __user *src, u32 size)
1198 {
1199         struct vpu_service_info *pservice = data->pservice;
1200         int extra_size = 0;
1201         struct extra_info_for_iommu extra_info;
1202         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1203                                       GFP_KERNEL);
1204
1205         vpu_debug_enter();
1206
1207         if (!reg) {
1208                 vpu_err("error: kzalloc failed\n");
1209                 return NULL;
1210         }
1211
1212         if (size > data->reg_size) {
1213                 extra_size = size - data->reg_size;
1214                 size = data->reg_size;
1215         }
1216         reg->session = session;
1217         reg->data = data;
1218         reg->type = session->type;
1219         reg->size = size;
1220         reg->freq = VPU_FREQ_DEFAULT;
1221         reg->task = &data->task_info[session->type];
1222         reg->trans = data->trans_info;
1223         reg->reg = (u32 *)&reg[1];
1224         INIT_LIST_HEAD(&reg->session_link);
1225         INIT_LIST_HEAD(&reg->status_link);
1226
1227         INIT_LIST_HEAD(&reg->mem_region_list);
1228
1229         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1230                 vpu_err("error: copy_from_user failed\n");
1231                 kfree(reg);
1232                 return NULL;
1233         }
1234
1235         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1236                 vpu_err("error: copy_from_user failed\n");
1237                 kfree(reg);
1238                 return NULL;
1239         }
1240
1241         if (vcodec_reg_address_translate(data, session, reg, &extra_info) < 0) {
1242                 int i = 0;
1243
1244                 vpu_err("error: translate reg address failed, dumping regs\n");
1245                 for (i = 0; i < size >> 2; i++)
1246                         dev_err(pservice->dev, "reg[%02d]: %08x\n",
1247                                 i, *((u32 *)src + i));
1248
1249                 kfree(reg);
1250                 return NULL;
1251         }
1252
1253         mutex_lock(&pservice->lock);
1254         list_add_tail(&reg->status_link, &pservice->waiting);
1255         list_add_tail(&reg->session_link, &session->waiting);
1256         mutex_unlock(&pservice->lock);
1257
1258         if (pservice->auto_freq)
1259                 get_reg_freq(data, reg);
1260
1261         vpu_debug_leave();
1262
1263         return reg;
1264 }
1265
1266 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1267 {
1268         struct vpu_service_info *pservice = data->pservice;
1269         struct vcodec_mem_region *mem_region = NULL, *n;
1270
1271         list_del_init(&reg->session_link);
1272         list_del_init(&reg->status_link);
1273         if (reg == pservice->reg_codec)
1274                 pservice->reg_codec = NULL;
1275         if (reg == pservice->reg_pproc)
1276                 pservice->reg_pproc = NULL;
1277
1278         /* release memory region attach to this registers table. */
1279         list_for_each_entry_safe(mem_region, n,
1280                         &reg->mem_region_list, reg_lnk) {
1281                 vcodec_iommu_unmap_iommu(data->iommu_info, reg->session,
1282                                          mem_region->hdl);
1283                 vcodec_iommu_free(data->iommu_info, reg->session,
1284                                   mem_region->hdl);
1285                 list_del_init(&mem_region->reg_lnk);
1286                 kfree(mem_region);
1287         }
1288
1289         kfree(reg);
1290 }
1291
1292 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1293                                  struct vpu_reg *reg)
1294 {
1295         vpu_debug_enter();
1296         list_del_init(&reg->status_link);
1297         list_add_tail(&reg->status_link, &pservice->running);
1298
1299         list_del_init(&reg->session_link);
1300         list_add_tail(&reg->session_link, &reg->session->running);
1301         vpu_debug_leave();
1302 }
1303
1304 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1305 {
1306         int i;
1307         u32 *dst = reg->reg;
1308
1309         vpu_debug_enter();
1310         for (i = 0; i < count; i++, src++)
1311                 *dst++ = readl_relaxed(src);
1312
1313         dst = (u32 *)&reg->reg[0];
1314         for (i = 0; i < count; i++)
1315                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1316
1317         vpu_debug_leave();
1318 }
1319
1320 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1321                                  struct vpu_reg *reg)
1322 {
1323         struct vpu_service_info *pservice = data->pservice;
1324         struct vpu_hw_info *hw_info = data->hw_info;
1325         struct vpu_task_info *task = reg->task;
1326
1327         vpu_debug_enter();
1328
1329         list_del_init(&reg->status_link);
1330         list_add_tail(&reg->status_link, &pservice->done);
1331
1332         list_del_init(&reg->session_link);
1333         list_add_tail(&reg->session_link, &reg->session->done);
1334
1335         switch (reg->type) {
1336         case VPU_ENC: {
1337                 pservice->reg_codec = NULL;
1338                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1339                 reg->reg[task->reg_irq] = pservice->irq_status;
1340         } break;
1341         case VPU_DEC: {
1342                 pservice->reg_codec = NULL;
1343                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1344
1345                 /* revert hack for decoded length */
1346                 if (task->reg_len > 0) {
1347                         int reg_len = task->reg_len;
1348                         u32 dec_get = reg->reg[reg_len];
1349                         s32 dec_length = dec_get - reg->dec_base;
1350
1351                         vpu_debug(DEBUG_REGISTER,
1352                                   "dec_get %08x dec_length %d\n",
1353                                   dec_get, dec_length);
1354                         reg->reg[reg_len] = dec_length << 10;
1355                 }
1356
1357                 reg->reg[task->reg_irq] = pservice->irq_status;
1358         } break;
1359         case VPU_PP: {
1360                 pservice->reg_pproc = NULL;
1361                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1362                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1363         } break;
1364         case VPU_DEC_PP: {
1365                 u32 pipe_mode;
1366                 u32 *regs = data->dec_dev.regs;
1367
1368                 pservice->reg_codec = NULL;
1369                 pservice->reg_pproc = NULL;
1370
1371                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1372
1373                 /* NOTE: remove pp pipeline mode flag first */
1374                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1375                 pipe_mode &= ~task->pipe_mask;
1376                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1377
1378                 /* revert hack for decoded length */
1379                 if (task->reg_len > 0) {
1380                         int reg_len = task->reg_len;
1381                         u32 dec_get = reg->reg[reg_len];
1382                         s32 dec_length = dec_get - reg->dec_base;
1383
1384                         vpu_debug(DEBUG_REGISTER,
1385                                   "dec_get %08x dec_length %d\n",
1386                                   dec_get, dec_length);
1387                         reg->reg[reg_len] = dec_length << 10;
1388                 }
1389
1390                 reg->reg[task->reg_irq] = pservice->irq_status;
1391         } break;
1392         default: {
1393                 vpu_err("error: copy reg from hw with unknown type %d\n",
1394                         reg->type);
1395         } break;
1396         }
1397         vcodec_exit_mode(data);
1398
1399         atomic_sub(1, &reg->session->task_running);
1400         atomic_sub(1, &pservice->total_running);
1401         wake_up(&reg->session->wait);
1402
1403         vpu_debug_leave();
1404 }
1405
1406 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1407                                  struct vpu_reg *reg)
1408 {
1409         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1410
1411         if (curr == reg->freq)
1412                 return;
1413
1414         atomic_set(&pservice->freq_status, reg->freq);
1415         switch (reg->freq) {
1416         case VPU_FREQ_200M: {
1417                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1418         } break;
1419         case VPU_FREQ_266M: {
1420                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1421         } break;
1422         case VPU_FREQ_300M: {
1423                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1424         } break;
1425         case VPU_FREQ_400M: {
1426                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1427         } break;
1428         case VPU_FREQ_500M: {
1429                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1430         } break;
1431         case VPU_FREQ_600M: {
1432                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1433         } break;
1434         default: {
1435                 unsigned long rate = 300*MHZ;
1436
1437                 if (of_machine_is_compatible("rockchip,rk2928g"))
1438                         rate = 400*MHZ;
1439
1440                 clk_set_rate(pservice->aclk_vcodec, rate);
1441         } break;
1442         }
1443 }
1444
1445 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1446 {
1447         struct vpu_service_info *pservice = data->pservice;
1448         struct vpu_task_info *task = reg->task;
1449         struct vpu_hw_info *hw_info = data->hw_info;
1450         int i;
1451         u32 *src = (u32 *)&reg->reg[0];
1452         u32 enable_mask = task->enable_mask;
1453         u32 gating_mask = task->gating_mask;
1454         u32 reg_en = task->reg_en;
1455
1456         vpu_debug_enter();
1457
1458         atomic_add(1, &pservice->total_running);
1459         atomic_add(1, &reg->session->task_running);
1460
1461         if (pservice->auto_freq)
1462                 vpu_service_set_freq(pservice, reg);
1463
1464         vcodec_enter_mode(data);
1465
1466         switch (reg->type) {
1467         case VPU_ENC: {
1468                 u32 *dst = data->enc_dev.regs;
1469                 u32 base = 0;
1470                 u32 end  = hw_info->enc_reg_num;
1471                 /* u32 reg_gating = task->reg_gating; */
1472
1473                 pservice->reg_codec = reg;
1474
1475                 vpu_debug(DEBUG_TASK_INFO,
1476                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1477                           base, end, reg_en, enable_mask, gating_mask);
1478
1479                 VEPU_CLEAN_CACHE(dst);
1480
1481                 if (debug & DEBUG_SET_REG)
1482                         for (i = base; i < end; i++)
1483                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1484                                           i, src[i]);
1485
1486                 /*
1487                  * NOTE: encoder need to setup mode first
1488                  */
1489                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1490
1491                 /* NOTE: encoder gating is not on enable register */
1492                 /* src[reg_gating] |= gating_mask; */
1493
1494                 for (i = base; i < end; i++) {
1495                         if (i != reg_en)
1496                                 writel_relaxed(src[i], dst + i);
1497                 }
1498
1499                 writel(src[reg_en], dst + reg_en);
1500                 dsb(sy);
1501
1502                 time_record(reg->task, 0);
1503         } break;
1504         case VPU_DEC: {
1505                 u32 *dst = data->dec_dev.regs;
1506                 u32 len = hw_info->dec_reg_num;
1507                 u32 base = hw_info->base_dec;
1508                 u32 end  = hw_info->end_dec;
1509
1510                 pservice->reg_codec = reg;
1511
1512                 vpu_debug(DEBUG_TASK_INFO,
1513                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1514                           base, end, reg_en, enable_mask, gating_mask);
1515
1516                 VDPU_CLEAN_CACHE(dst);
1517
1518                 /* on rkvdec set cache size to 64byte */
1519                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1520                         u32 *cache_base = dst + 0x100;
1521                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1522                         writel_relaxed(val, cache_base + 0x07);
1523                         writel_relaxed(val, cache_base + 0x17);
1524                 }
1525
1526                 if (debug & DEBUG_SET_REG)
1527                         for (i = 0; i < len; i++)
1528                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1529                                           i, src[i]);
1530                 /*
1531                  * NOTE: The end register is invalid. Do NOT write to it
1532                  *       Also the base register must be written
1533                  */
1534                 for (i = base; i < end; i++) {
1535                         if (i != reg_en)
1536                                 writel_relaxed(src[i], dst + i);
1537                 }
1538
1539                 writel(src[reg_en] | gating_mask, dst + reg_en);
1540                 dsb(sy);
1541
1542                 time_record(reg->task, 0);
1543         } break;
1544         case VPU_PP: {
1545                 u32 *dst = data->dec_dev.regs;
1546                 u32 base = hw_info->base_pp;
1547                 u32 end  = hw_info->end_pp;
1548
1549                 pservice->reg_pproc = reg;
1550
1551                 vpu_debug(DEBUG_TASK_INFO,
1552                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1553                           base, end, reg_en, enable_mask, gating_mask);
1554
1555                 if (debug & DEBUG_SET_REG)
1556                         for (i = base; i < end; i++)
1557                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1558                                           i, src[i]);
1559
1560                 for (i = base; i < end; i++) {
1561                         if (i != reg_en)
1562                                 writel_relaxed(src[i], dst + i);
1563                 }
1564
1565                 writel(src[reg_en] | gating_mask, dst + reg_en);
1566                 dsb(sy);
1567
1568                 time_record(reg->task, 0);
1569         } break;
1570         case VPU_DEC_PP: {
1571                 u32 *dst = data->dec_dev.regs;
1572                 u32 base = hw_info->base_dec_pp;
1573                 u32 end  = hw_info->end_dec_pp;
1574
1575                 pservice->reg_codec = reg;
1576                 pservice->reg_pproc = reg;
1577
1578                 vpu_debug(DEBUG_TASK_INFO,
1579                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1580                           base, end, reg_en, enable_mask, gating_mask);
1581
1582                 /* VDPU_SOFT_RESET(dst); */
1583                 VDPU_CLEAN_CACHE(dst);
1584
1585                 if (debug & DEBUG_SET_REG)
1586                         for (i = base; i < end; i++)
1587                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1588                                           i, src[i]);
1589
1590                 for (i = base; i < end; i++) {
1591                         if (i != reg_en)
1592                                 writel_relaxed(src[i], dst + i);
1593                 }
1594
1595                 /* NOTE: dec output must be disabled */
1596
1597                 writel(src[reg_en] | gating_mask, dst + reg_en);
1598                 dsb(sy);
1599
1600                 time_record(reg->task, 0);
1601         } break;
1602         default: {
1603                 vpu_err("error: unsupport session type %d", reg->type);
1604                 atomic_sub(1, &pservice->total_running);
1605                 atomic_sub(1, &reg->session->task_running);
1606         } break;
1607         }
1608
1609         vpu_debug_leave();
1610 }
1611
1612 static void try_set_reg(struct vpu_subdev_data *data)
1613 {
1614         struct vpu_service_info *pservice = data->pservice;
1615
1616         vpu_debug_enter();
1617
1618         mutex_lock(&pservice->shutdown_lock);
1619         if (atomic_read(&pservice->service_on) == 0) {
1620                 mutex_unlock(&pservice->shutdown_lock);
1621                 return;
1622         }
1623         if (!list_empty(&pservice->waiting)) {
1624                 struct vpu_reg *reg_codec = pservice->reg_codec;
1625                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1626                 int can_set = 0;
1627                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1628                 int reset_request = atomic_read(&pservice->reset_request);
1629                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1630                                 struct vpu_reg, status_link);
1631
1632                 vpu_service_power_on(data, pservice);
1633
1634                 if (change_able || !reset_request) {
1635                         switch (reg->type) {
1636                         case VPU_ENC: {
1637                                 if (change_able)
1638                                         can_set = 1;
1639                         } break;
1640                         case VPU_DEC: {
1641                                 if (reg_codec == NULL)
1642                                         can_set = 1;
1643                                 if (pservice->auto_freq && (reg_pproc != NULL))
1644                                         can_set = 0;
1645                         } break;
1646                         case VPU_PP: {
1647                                 if (reg_codec == NULL) {
1648                                         if (reg_pproc == NULL)
1649                                                 can_set = 1;
1650                                 } else {
1651                                         if ((reg_codec->type == VPU_DEC) &&
1652                                             (reg_pproc == NULL))
1653                                                 can_set = 1;
1654
1655                                         /*
1656                                          * NOTE:
1657                                          * can not charge frequency
1658                                          * when vpu is working
1659                                          */
1660                                         if (pservice->auto_freq)
1661                                                 can_set = 0;
1662                                 }
1663                         } break;
1664                         case VPU_DEC_PP: {
1665                                 if (change_able)
1666                                         can_set = 1;
1667                                 } break;
1668                         default: {
1669                                 dev_err(pservice->dev,
1670                                         "undefined reg type %d\n",
1671                                         reg->type);
1672                         } break;
1673                         }
1674                 }
1675
1676                 /* then check reset request */
1677                 if (reset_request && !change_able)
1678                         reset_request = 0;
1679
1680                 /* do reset before setting registers */
1681                 if (reset_request)
1682                         vpu_reset(data);
1683
1684                 if (can_set) {
1685                         reg_from_wait_to_run(pservice, reg);
1686                         reg_copy_to_hw(reg->data, reg);
1687                 }
1688         }
1689
1690         mutex_unlock(&pservice->shutdown_lock);
1691         vpu_debug_leave();
1692 }
1693
1694 static void vpu_set_register_work(struct work_struct *work_s)
1695 {
1696         struct vpu_subdev_data *data = container_of(work_s,
1697                                                     struct vpu_subdev_data,
1698                                                     set_work);
1699         struct vpu_service_info *pservice = data->pservice;
1700
1701         mutex_lock(&pservice->lock);
1702         try_set_reg(data);
1703         mutex_unlock(&pservice->lock);
1704 }
1705
1706 static int return_reg(struct vpu_subdev_data *data,
1707                       struct vpu_reg *reg, u32 __user *dst)
1708 {
1709         struct vpu_hw_info *hw_info = data->hw_info;
1710         size_t size = reg->size;
1711         u32 base;
1712
1713         vpu_debug_enter();
1714         switch (reg->type) {
1715         case VPU_ENC: {
1716                 base = 0;
1717         } break;
1718         case VPU_DEC: {
1719                 base = hw_info->base_dec_pp;
1720         } break;
1721         case VPU_PP: {
1722                 base = hw_info->base_pp;
1723         } break;
1724         case VPU_DEC_PP: {
1725                 base = hw_info->base_dec_pp;
1726         } break;
1727         default: {
1728                 vpu_err("error: copy reg to user with unknown type %d\n",
1729                         reg->type);
1730                 return -EFAULT;
1731         } break;
1732         }
1733
1734         if (copy_to_user(dst, &reg->reg[base], size)) {
1735                 vpu_err("error: copy_to_user failed\n");
1736                 return -EFAULT;
1737         }
1738
1739         reg_deinit(data, reg);
1740         vpu_debug_leave();
1741         return 0;
1742 }
1743
1744 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1745                               unsigned long arg)
1746 {
1747         struct vpu_subdev_data *data =
1748                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1749                              struct vpu_subdev_data, cdev);
1750         struct vpu_service_info *pservice = data->pservice;
1751         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1752
1753         vpu_debug_enter();
1754         if (NULL == session)
1755                 return -EINVAL;
1756
1757         switch (cmd) {
1758         case VPU_IOC_SET_CLIENT_TYPE: {
1759                 session->type = (enum VPU_CLIENT_TYPE)arg;
1760                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1761                           session->pid, session->type);
1762         } break;
1763         case VPU_IOC_GET_HW_FUSE_STATUS: {
1764                 struct vpu_request req;
1765
1766                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1767                           session->pid, session->type);
1768                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1769                         vpu_err("error: get hw status copy_from_user failed\n");
1770                         return -EFAULT;
1771                 } else {
1772                         void *config = (session->type != VPU_ENC) ?
1773                                        ((void *)&pservice->dec_config) :
1774                                        ((void *)&pservice->enc_config);
1775                         size_t size = (session->type != VPU_ENC) ?
1776                                       (sizeof(struct vpu_dec_config)) :
1777                                       (sizeof(struct vpu_enc_config));
1778                         if (copy_to_user((void __user *)req.req,
1779                                          config, size)) {
1780                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1781                                         session->type);
1782                                 return -EFAULT;
1783                         }
1784                 }
1785         } break;
1786         case VPU_IOC_SET_REG: {
1787                 struct vpu_request req;
1788                 struct vpu_reg *reg;
1789
1790                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1791                           session->pid, session->type);
1792                 if (copy_from_user(&req, (void __user *)arg,
1793                                    sizeof(struct vpu_request))) {
1794                         vpu_err("error: set reg copy_from_user failed\n");
1795                         return -EFAULT;
1796                 }
1797
1798                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1799                 if (NULL == reg) {
1800                         return -EFAULT;
1801                 } else {
1802                         queue_work(pservice->set_workq, &data->set_work);
1803                 }
1804         } break;
1805         case VPU_IOC_GET_REG: {
1806                 struct vpu_request req;
1807                 struct vpu_reg *reg;
1808                 int ret;
1809
1810                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1811                           session->pid, session->type);
1812                 if (copy_from_user(&req, (void __user *)arg,
1813                                    sizeof(struct vpu_request))) {
1814                         vpu_err("error: get reg copy_from_user failed\n");
1815                         return -EFAULT;
1816                 }
1817
1818                 ret = wait_event_timeout(session->wait,
1819                                          !list_empty(&session->done),
1820                                          VPU_TIMEOUT_DELAY);
1821
1822                 if (!list_empty(&session->done)) {
1823                         if (ret < 0)
1824                                 vpu_err("warning: pid %d wait task error ret %d\n",
1825                                         session->pid, ret);
1826                         ret = 0;
1827                 } else {
1828                         if (unlikely(ret < 0)) {
1829                                 vpu_err("error: pid %d wait task ret %d\n",
1830                                         session->pid, ret);
1831                         } else if (ret == 0) {
1832                                 vpu_err("error: pid %d wait %d task done timeout\n",
1833                                         session->pid,
1834                                         atomic_read(&session->task_running));
1835                                 ret = -ETIMEDOUT;
1836                         }
1837                 }
1838
1839                 if (ret < 0) {
1840                         int task_running = atomic_read(&session->task_running);
1841
1842                         mutex_lock(&pservice->lock);
1843                         vpu_service_dump(pservice);
1844                         if (task_running) {
1845                                 atomic_set(&session->task_running, 0);
1846                                 atomic_sub(task_running,
1847                                            &pservice->total_running);
1848                                 dev_err(pservice->dev,
1849                                         "%d task is running but not return, reset hardware...",
1850                                         task_running);
1851                                 vpu_reset(data);
1852                                 dev_err(pservice->dev, "done\n");
1853                         }
1854                         vpu_service_session_clear(data, session);
1855                         mutex_unlock(&pservice->lock);
1856
1857                         return ret;
1858                 }
1859                 mutex_lock(&pservice->lock);
1860                 reg = list_entry(session->done.next,
1861                                  struct vpu_reg, session_link);
1862                 return_reg(data, reg, (u32 __user *)req.req);
1863                 mutex_unlock(&pservice->lock);
1864         } break;
1865         case VPU_IOC_PROBE_IOMMU_STATUS: {
1866                 int iommu_enable = 1;
1867
1868                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1869
1870                 if (copy_to_user((void __user *)arg,
1871                                  &iommu_enable, sizeof(int))) {
1872                         vpu_err("error: iommu status copy_to_user failed\n");
1873                         return -EFAULT;
1874                 }
1875         } break;
1876         default: {
1877                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1878         } break;
1879         }
1880         vpu_debug_leave();
1881         return 0;
1882 }
1883
1884 #ifdef CONFIG_COMPAT
1885 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1886                                      unsigned long arg)
1887 {
1888         struct vpu_subdev_data *data =
1889                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1890                              struct vpu_subdev_data, cdev);
1891         struct vpu_service_info *pservice = data->pservice;
1892         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1893
1894         vpu_debug_enter();
1895         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1896                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1897         if (NULL == session)
1898                 return -EINVAL;
1899
1900         switch (cmd) {
1901         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1902                 session->type = (enum VPU_CLIENT_TYPE)arg;
1903                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1904                           session->type);
1905         } break;
1906         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1907                 struct compat_vpu_request req;
1908
1909                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1910                           session->type);
1911                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1912                                    sizeof(struct compat_vpu_request))) {
1913                         vpu_err("error: compat get hw status copy_from_user failed\n");
1914                         return -EFAULT;
1915                 } else {
1916                         void *config = (session->type != VPU_ENC) ?
1917                                        ((void *)&pservice->dec_config) :
1918                                        ((void *)&pservice->enc_config);
1919                         size_t size = (session->type != VPU_ENC) ?
1920                                       (sizeof(struct vpu_dec_config)) :
1921                                       (sizeof(struct vpu_enc_config));
1922
1923                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1924                                          config, size)) {
1925                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1926                                         session->type);
1927                                 return -EFAULT;
1928                         }
1929                 }
1930         } break;
1931         case COMPAT_VPU_IOC_SET_REG: {
1932                 struct compat_vpu_request req;
1933                 struct vpu_reg *reg;
1934
1935                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1936                           session->type);
1937                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1938                                    sizeof(struct compat_vpu_request))) {
1939                         vpu_err("compat set_reg copy_from_user failed\n");
1940                         return -EFAULT;
1941                 }
1942                 reg = reg_init(data, session,
1943                                compat_ptr((compat_uptr_t)req.req), req.size);
1944                 if (NULL == reg) {
1945                         return -EFAULT;
1946                 } else {
1947                         queue_work(pservice->set_workq, &data->set_work);
1948                 }
1949         } break;
1950         case COMPAT_VPU_IOC_GET_REG: {
1951                 struct compat_vpu_request req;
1952                 struct vpu_reg *reg;
1953                 int ret;
1954
1955                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1956                           session->type);
1957                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1958                                    sizeof(struct compat_vpu_request))) {
1959                         vpu_err("compat get reg copy_from_user failed\n");
1960                         return -EFAULT;
1961                 }
1962
1963                 ret = wait_event_timeout(session->wait,
1964                                          !list_empty(&session->done),
1965                                          VPU_TIMEOUT_DELAY);
1966
1967                 if (!list_empty(&session->done)) {
1968                         if (ret < 0)
1969                                 vpu_err("warning: pid %d wait task error ret %d\n",
1970                                         session->pid, ret);
1971                         ret = 0;
1972                 } else {
1973                         if (unlikely(ret < 0)) {
1974                                 vpu_err("error: pid %d wait task ret %d\n",
1975                                         session->pid, ret);
1976                         } else if (ret == 0) {
1977                                 vpu_err("error: pid %d wait %d task done timeout\n",
1978                                         session->pid,
1979                                         atomic_read(&session->task_running));
1980                                 ret = -ETIMEDOUT;
1981                         }
1982                 }
1983
1984                 if (ret < 0) {
1985                         int task_running = atomic_read(&session->task_running);
1986
1987                         mutex_lock(&pservice->lock);
1988                         vpu_service_dump(pservice);
1989                         if (task_running) {
1990                                 atomic_set(&session->task_running, 0);
1991                                 atomic_sub(task_running,
1992                                            &pservice->total_running);
1993                                 dev_err(pservice->dev,
1994                                         "%d task is running but not return, reset hardware...",
1995                                         task_running);
1996                                 vpu_reset(data);
1997                                 dev_err(pservice->dev, "done\n");
1998                         }
1999                         vpu_service_session_clear(data, session);
2000                         mutex_unlock(&pservice->lock);
2001                         return ret;
2002                 }
2003
2004                 mutex_lock(&pservice->lock);
2005                 reg = list_entry(session->done.next,
2006                                  struct vpu_reg, session_link);
2007                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
2008                 mutex_unlock(&pservice->lock);
2009         } break;
2010         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
2011                 int iommu_enable = 1;
2012
2013                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
2014
2015                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
2016                                  &iommu_enable, sizeof(int))) {
2017                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2018                         return -EFAULT;
2019                 }
2020         } break;
2021         default: {
2022                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2023         } break;
2024         }
2025         vpu_debug_leave();
2026         return 0;
2027 }
2028 #endif
2029
2030 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2031 {
2032         struct vpu_service_info *pservice = data->pservice;
2033         int ret = -EINVAL, i = 0;
2034         u32 hw_id = readl_relaxed(data->regs);
2035
2036         hw_id = (hw_id >> 16) & 0xFFFF;
2037         dev_info(pservice->dev, "checking hw id %x\n", hw_id);
2038         data->hw_info = NULL;
2039
2040         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2041                 const struct vcodec_info *info = &vcodec_info_set[i];
2042
2043                 if (hw_id == info->hw_id) {
2044                         data->hw_id = info->hw_id;
2045                         data->hw_info = info->hw_info;
2046                         data->task_info = info->task_info;
2047                         data->trans_info = info->trans_info;
2048                         ret = 0;
2049                         break;
2050                 }
2051         }
2052         return ret;
2053 }
2054
2055 static int vpu_service_open(struct inode *inode, struct file *filp)
2056 {
2057         struct vpu_subdev_data *data = container_of(
2058                         inode->i_cdev, struct vpu_subdev_data, cdev);
2059         struct vpu_service_info *pservice = data->pservice;
2060         struct vpu_session *session = NULL;
2061
2062         vpu_debug_enter();
2063
2064         session = kzalloc(sizeof(*session), GFP_KERNEL);
2065         if (!session) {
2066                 vpu_err("error: unable to allocate memory for vpu_session.");
2067                 return -ENOMEM;
2068         }
2069
2070         data->iommu_info->debug_level = debug;
2071
2072         session->type   = VPU_TYPE_BUTT;
2073         session->pid    = current->pid;
2074         INIT_LIST_HEAD(&session->waiting);
2075         INIT_LIST_HEAD(&session->running);
2076         INIT_LIST_HEAD(&session->done);
2077         INIT_LIST_HEAD(&session->list_session);
2078         init_waitqueue_head(&session->wait);
2079         atomic_set(&session->task_running, 0);
2080         mutex_lock(&pservice->lock);
2081         list_add_tail(&session->list_session, &pservice->session);
2082         filp->private_data = (void *)session;
2083         mutex_unlock(&pservice->lock);
2084
2085         dev_dbg(pservice->dev, "dev opened\n");
2086         vpu_debug_leave();
2087         return nonseekable_open(inode, filp);
2088 }
2089
2090 static int vpu_service_release(struct inode *inode, struct file *filp)
2091 {
2092         struct vpu_subdev_data *data = container_of(
2093                         inode->i_cdev, struct vpu_subdev_data, cdev);
2094         struct vpu_service_info *pservice = data->pservice;
2095         int task_running;
2096         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2097
2098         vpu_debug_enter();
2099         if (NULL == session)
2100                 return -EINVAL;
2101
2102         task_running = atomic_read(&session->task_running);
2103         if (task_running) {
2104                 dev_err(pservice->dev,
2105                         "error: session %d still has %d task running when closing\n",
2106                         session->pid, task_running);
2107                 msleep(50);
2108         }
2109         wake_up(&session->wait);
2110
2111         mutex_lock(&pservice->lock);
2112         /* remove this filp from the asynchronusly notified filp's */
2113         list_del_init(&session->list_session);
2114         vpu_service_session_clear(data, session);
2115         vcodec_iommu_clear(data->iommu_info, session);
2116         kfree(session);
2117         filp->private_data = NULL;
2118         mutex_unlock(&pservice->lock);
2119
2120         dev_info(pservice->dev, "closed\n");
2121         vpu_debug_leave();
2122         return 0;
2123 }
2124
2125 static const struct file_operations vpu_service_fops = {
2126         .unlocked_ioctl = vpu_service_ioctl,
2127         .open           = vpu_service_open,
2128         .release        = vpu_service_release,
2129 #ifdef CONFIG_COMPAT
2130         .compat_ioctl   = compat_vpu_service_ioctl,
2131 #endif
2132 };
2133
2134 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2135 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2136 static irqreturn_t vepu_irq(int irq, void *dev_id);
2137 static irqreturn_t vepu_isr(int irq, void *dev_id);
2138 static void get_hw_info(struct vpu_subdev_data *data);
2139
2140 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2141 {
2142         struct device_node *dn = NULL;
2143         struct platform_device *pd = NULL;
2144         struct device *ret = NULL;
2145
2146         dn = of_find_compatible_node(NULL, NULL, compt);
2147         if (!dn) {
2148                 pr_err("can't find device node %s \r\n", compt);
2149                 return NULL;
2150         }
2151
2152         pd = of_find_device_by_node(dn);
2153         if (!pd) {
2154                 pr_err("can't find platform device in device node %s\n", compt);
2155                 return  NULL;
2156         }
2157         ret = &pd->dev;
2158
2159         return ret;
2160 }
2161
2162 #ifdef CONFIG_IOMMU_API
2163 static inline void platform_set_sysmmu(struct device *iommu,
2164                                        struct device *dev)
2165 {
2166         dev->archdata.iommu = iommu;
2167 }
2168 #else
2169 static inline void platform_set_sysmmu(struct device *iommu,
2170                                        struct device *dev)
2171 {
2172 }
2173 #endif
2174
2175 int vcodec_sysmmu_fault_hdl(struct device *dev,
2176                             enum rk_iommu_inttype itype,
2177                             unsigned long pgtable_base,
2178                             unsigned long fault_addr, unsigned int status)
2179 {
2180         struct platform_device *pdev;
2181         struct vpu_service_info *pservice;
2182         struct vpu_subdev_data *data;
2183
2184         vpu_debug_enter();
2185
2186         if (dev == NULL) {
2187                 pr_err("invalid NULL dev\n");
2188                 return 0;
2189         }
2190
2191         pdev = container_of(dev, struct platform_device, dev);
2192         if (pdev == NULL) {
2193                 pr_err("invalid NULL platform_device\n");
2194                 return 0;
2195         }
2196
2197         data = platform_get_drvdata(pdev);
2198         if (data == NULL) {
2199                 pr_err("invalid NULL vpu_subdev_data\n");
2200                 return 0;
2201         }
2202
2203         pservice = data->pservice;
2204         if (pservice == NULL) {
2205                 pr_err("invalid NULL vpu_service_info\n");
2206                 return 0;
2207         }
2208
2209         if (pservice->reg_codec) {
2210                 struct vpu_reg *reg = pservice->reg_codec;
2211                 struct vcodec_mem_region *mem, *n;
2212                 int i = 0;
2213
2214                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2215                 if (!list_empty(&reg->mem_region_list)) {
2216                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2217                                                  reg_lnk) {
2218                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2219                                        mem->reg_idx, i, mem->iova, mem->len);
2220                                 i++;
2221                         }
2222                 } else {
2223                         pr_err("no memory region mapped\n");
2224                 }
2225
2226                 if (reg->data) {
2227                         struct vpu_subdev_data *data = reg->data;
2228                         u32 *base = (u32 *)data->dec_dev.regs;
2229                         u32 len = data->hw_info->dec_reg_num;
2230
2231                         pr_err("current errror register set:\n");
2232
2233                         for (i = 0; i < len; i++)
2234                                 pr_err("reg[%02d] %08x\n",
2235                                        i, readl_relaxed(base + i));
2236                 }
2237
2238                 pr_alert("vcodec, page fault occur, reset hw\n");
2239
2240                 /* reg->reg[101] = 1; */
2241                 _vpu_reset(data);
2242         }
2243
2244         return 0;
2245 }
2246
2247 static int vcodec_subdev_probe(struct platform_device *pdev,
2248                                struct vpu_service_info *pservice)
2249 {
2250         uint8_t *regs = NULL;
2251         int32_t ret = 0;
2252         uint32_t ioaddr = 0;
2253         struct resource *res = NULL;
2254         struct vpu_hw_info *hw_info = NULL;
2255         struct device *dev = &pdev->dev;
2256         struct device_node *np = pdev->dev.of_node;
2257         struct vpu_subdev_data *data = NULL;
2258         struct platform_device *sub_dev = NULL;
2259         struct device_node *sub_np = NULL;
2260         const char *name  = np->name;
2261         char mmu_dev_dts_name[40];
2262
2263         dev_info(dev, "probe device");
2264
2265         data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2266         if (!data)
2267                 return -ENOMEM;
2268
2269         data->pservice = pservice;
2270         data->dev = dev;
2271
2272         INIT_WORK(&data->set_work, vpu_set_register_work);
2273         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2274
2275         if (pservice->reg_base == 0) {
2276                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2277                 data->regs = devm_ioremap_resource(dev, res);
2278                 if (IS_ERR(data->regs)) {
2279                         ret = PTR_ERR(data->regs);
2280                         goto err;
2281                 }
2282                 ioaddr = res->start;
2283         } else {
2284                 data->regs = pservice->reg_base;
2285                 ioaddr = pservice->ioaddr;
2286         }
2287
2288         sub_np = of_parse_phandle(np, "iommus", 0);
2289         if (sub_np) {
2290                 sub_dev = of_find_device_by_node(sub_np);
2291                 data->mmu_dev = &sub_dev->dev;
2292         }
2293
2294         /* Back to legacy iommu probe */
2295         if (!data->mmu_dev) {
2296                 switch (data->mode) {
2297                 case VCODEC_RUNNING_MODE_VPU:
2298                         sprintf(mmu_dev_dts_name,
2299                                 VPU_IOMMU_COMPATIBLE_NAME);
2300                         break;
2301                 case VCODEC_RUNNING_MODE_RKVDEC:
2302                         sprintf(mmu_dev_dts_name,
2303                                 VDEC_IOMMU_COMPATIBLE_NAME);
2304                         break;
2305                 case VCODEC_RUNNING_MODE_HEVC:
2306                 default:
2307                         sprintf(mmu_dev_dts_name,
2308                                 HEVC_IOMMU_COMPATIBLE_NAME);
2309                         break;
2310                 }
2311
2312                 data->mmu_dev =
2313                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2314                 if (data->mmu_dev)
2315                         platform_set_sysmmu(data->mmu_dev, dev);
2316
2317                 rockchip_iovmm_set_fault_handler
2318                         (dev, vcodec_sysmmu_fault_hdl);
2319         }
2320
2321         dev_info(dev, "vpu mmu dec %p\n", data->mmu_dev);
2322
2323         clear_bit(MMU_ACTIVATED, &data->state);
2324         vpu_service_power_on(data, pservice);
2325
2326         of_property_read_u32(np, "allocator", (u32 *)&pservice->alloc_type);
2327         data->iommu_info = vcodec_iommu_info_create(dev, data->mmu_dev,
2328                                                     pservice->alloc_type);
2329         dev_info(dev, "allocator is %s\n", pservice->alloc_type == 1 ? "drm" :
2330                 (pservice->alloc_type == 2 ? "ion" : "null"));
2331         vcodec_enter_mode(data);
2332         ret = vpu_service_check_hw(data);
2333         if (ret < 0) {
2334                 vpu_err("error: hw info check faild\n");
2335                 goto err;
2336         }
2337         vcodec_exit_mode(data);
2338
2339         hw_info = data->hw_info;
2340         regs = (u8 *)data->regs;
2341
2342         if (hw_info->dec_reg_num) {
2343                 data->dec_dev.iosize = hw_info->dec_io_size;
2344                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2345         }
2346
2347         if (hw_info->enc_reg_num) {
2348                 data->enc_dev.iosize = hw_info->enc_io_size;
2349                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2350         }
2351
2352         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2353
2354         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2355         if (data->irq_enc > 0) {
2356                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2357                                                 vepu_irq, vepu_isr,
2358                                                 IRQF_SHARED, dev_name(dev),
2359                                                 (void *)data);
2360                 if (ret) {
2361                         dev_err(dev, "error: can't request vepu irq %d\n",
2362                                 data->irq_enc);
2363                         goto err;
2364                 }
2365         }
2366         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2367         if (data->irq_dec > 0) {
2368                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2369                                                 vdpu_irq, vdpu_isr,
2370                                                 IRQF_SHARED, dev_name(dev),
2371                                                 (void *)data);
2372                 if (ret) {
2373                         dev_err(dev, "error: can't request vdpu irq %d\n",
2374                                 data->irq_dec);
2375                         goto err;
2376                 }
2377         }
2378         atomic_set(&data->dec_dev.irq_count_codec, 0);
2379         atomic_set(&data->dec_dev.irq_count_pp, 0);
2380         atomic_set(&data->enc_dev.irq_count_codec, 0);
2381         atomic_set(&data->enc_dev.irq_count_pp, 0);
2382
2383         get_hw_info(data);
2384         pservice->auto_freq = true;
2385
2386         /* create device node */
2387         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2388         if (ret) {
2389                 dev_err(dev, "alloc dev_t failed\n");
2390                 goto err;
2391         }
2392
2393         cdev_init(&data->cdev, &vpu_service_fops);
2394
2395         data->cdev.owner = THIS_MODULE;
2396         data->cdev.ops = &vpu_service_fops;
2397
2398         ret = cdev_add(&data->cdev, data->dev_t, 1);
2399
2400         if (ret) {
2401                 dev_err(dev, "add dev_t failed\n");
2402                 goto err;
2403         }
2404
2405         data->cls = class_create(THIS_MODULE, name);
2406
2407         if (IS_ERR(data->cls)) {
2408                 ret = PTR_ERR(data->cls);
2409                 dev_err(dev, "class_create err:%d\n", ret);
2410                 goto err;
2411         }
2412
2413         data->child_dev = device_create(data->cls, dev,
2414                 data->dev_t, "%s", name);
2415
2416         platform_set_drvdata(pdev, data);
2417
2418         INIT_LIST_HEAD(&data->lnk_service);
2419         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2420
2421         return 0;
2422 err:
2423         if (data->child_dev) {
2424                 device_destroy(data->cls, data->dev_t);
2425                 cdev_del(&data->cdev);
2426                 unregister_chrdev_region(data->dev_t, 1);
2427         }
2428
2429         if (data->cls)
2430                 class_destroy(data->cls);
2431         return -1;
2432 }
2433
2434 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2435 {
2436         struct vpu_service_info *pservice = data->pservice;
2437
2438         vcodec_iommu_info_destroy(data->iommu_info);
2439         data->iommu_info = NULL;
2440
2441         mutex_lock(&pservice->lock);
2442         cancel_delayed_work_sync(&pservice->power_off_work);
2443         vpu_service_power_off(pservice);
2444         mutex_unlock(&pservice->lock);
2445
2446         device_destroy(data->cls, data->dev_t);
2447         class_destroy(data->cls);
2448         cdev_del(&data->cdev);
2449         unregister_chrdev_region(data->dev_t, 1);
2450
2451 #ifdef CONFIG_DEBUG_FS
2452         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2453                 debugfs_remove_recursive(data->debugfs_dir);
2454 #endif
2455 }
2456
2457 static void vcodec_read_property(struct device_node *np,
2458                                  struct vpu_service_info *pservice)
2459 {
2460         pservice->mode_bit = 0;
2461         pservice->mode_ctrl = 0;
2462         pservice->subcnt = 0;
2463         pservice->grf_base = NULL;
2464
2465         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2466
2467         if (pservice->subcnt > 1) {
2468                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2469                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2470         }
2471 #ifdef CONFIG_MFD_SYSCON
2472         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2473         if (IS_ERR_OR_NULL(pservice->grf)) {
2474                 pservice->grf = NULL;
2475 #ifdef CONFIG_ARM
2476                 pservice->grf_base = RK_GRF_VIRT;
2477 #else
2478                 vpu_err("can't find vpu grf property\n");
2479                 return;
2480 #endif
2481         }
2482 #else
2483 #ifdef CONFIG_ARM
2484         pservice->grf_base = RK_GRF_VIRT;
2485 #else
2486         vpu_err("can't find vpu grf property\n");
2487         return;
2488 #endif
2489 #endif
2490
2491 #ifdef CONFIG_RESET_CONTROLLER
2492         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2493         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2494         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2495
2496         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2497                 dev_warn(pservice->dev, "No aclk reset resource define\n");
2498                 pservice->rst_a = NULL;
2499         }
2500
2501         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2502                 dev_warn(pservice->dev, "No hclk reset resource define\n");
2503                 pservice->rst_h = NULL;
2504         }
2505
2506         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2507                 dev_warn(pservice->dev, "No core reset resource define\n");
2508                 pservice->rst_v = NULL;
2509         }
2510 #endif
2511
2512         of_property_read_string(np, "name", (const char **)&pservice->name);
2513 }
2514
2515 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2516 {
2517         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2518         pservice->curr_mode = -1;
2519
2520         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2521         INIT_LIST_HEAD(&pservice->waiting);
2522         INIT_LIST_HEAD(&pservice->running);
2523         mutex_init(&pservice->lock);
2524         mutex_init(&pservice->shutdown_lock);
2525         atomic_set(&pservice->service_on, 1);
2526
2527         INIT_LIST_HEAD(&pservice->done);
2528         INIT_LIST_HEAD(&pservice->session);
2529         INIT_LIST_HEAD(&pservice->subdev_list);
2530
2531         pservice->reg_pproc     = NULL;
2532         atomic_set(&pservice->total_running, 0);
2533         atomic_set(&pservice->enabled,       0);
2534         atomic_set(&pservice->power_on_cnt,  0);
2535         atomic_set(&pservice->power_off_cnt, 0);
2536         atomic_set(&pservice->reset_request, 0);
2537
2538         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2539         pservice->last.tv64 = 0;
2540
2541         pservice->alloc_type = 0;
2542 }
2543
2544 static int vcodec_probe(struct platform_device *pdev)
2545 {
2546         int i;
2547         int ret = 0;
2548         struct resource *res = NULL;
2549         struct device *dev = &pdev->dev;
2550         struct device_node *np = pdev->dev.of_node;
2551         struct vpu_service_info *pservice = NULL;
2552         struct vcodec_device_info *driver_data;
2553
2554         pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info),
2555                                 GFP_KERNEL);
2556         if (!pservice)
2557                 return -ENOMEM;
2558         pservice->dev = dev;
2559
2560         pservice->set_workq = create_singlethread_workqueue("vcodec");
2561         if (!pservice->set_workq) {
2562                 dev_err(dev, "failed to create workqueue\n");
2563                 return -ENOMEM;
2564         }
2565
2566         driver_data = vcodec_get_drv_data(pdev);
2567         if (!driver_data)
2568                 return -EINVAL;
2569
2570         vcodec_read_property(np, pservice);
2571         vcodec_init_drvdata(pservice);
2572
2573         /* Underscore for label, hyphens for name */
2574         switch (driver_data->device_type) {
2575         case VCODEC_DEVICE_TYPE_VPUX:
2576                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2577                 break;
2578         case VCODEC_DEVICE_TYPE_VPUC:
2579                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2580                 break;
2581         case VCODEC_DEVICE_TYPE_HEVC:
2582                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2583                 break;
2584         case VCODEC_DEVICE_TYPE_RKVD:
2585                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2586                 break;
2587         default:
2588                 dev_err(dev, "unsupported device type\n");
2589                 return -ENODEV;
2590         }
2591
2592         if (0 > vpu_get_clk(pservice))
2593                 goto err;
2594
2595         if (of_property_read_bool(np, "reg")) {
2596                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2597
2598                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2599                 if (IS_ERR(pservice->reg_base)) {
2600                         vpu_err("ioremap registers base failed\n");
2601                         ret = PTR_ERR(pservice->reg_base);
2602                         goto err;
2603                 }
2604                 pservice->ioaddr = res->start;
2605         } else {
2606                 pservice->reg_base = 0;
2607         }
2608
2609         pm_runtime_enable(dev);
2610
2611         if (of_property_read_bool(np, "subcnt")) {
2612                 struct vpu_subdev_data *data = NULL;
2613
2614                 data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data),
2615                                     GFP_KERNEL);
2616                 if (!data)
2617                         return -ENOMEM;
2618
2619                 for (i = 0; i < pservice->subcnt; i++) {
2620                         struct device_node *sub_np;
2621                         struct platform_device *sub_pdev;
2622
2623                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2624                         sub_pdev = of_find_device_by_node(sub_np);
2625
2626                         vcodec_subdev_probe(sub_pdev, pservice);
2627                 }
2628                 data->pservice = pservice;
2629                 platform_set_drvdata(pdev, data);
2630         } else {
2631                 vcodec_subdev_probe(pdev, pservice);
2632         }
2633
2634         vpu_service_power_off(pservice);
2635
2636         dev_info(dev, "init success\n");
2637
2638         return 0;
2639
2640 err:
2641         dev_info(dev, "init failed\n");
2642         vpu_service_power_off(pservice);
2643         destroy_workqueue(pservice->set_workq);
2644         wake_lock_destroy(&pservice->wake_lock);
2645
2646         return ret;
2647 }
2648
2649 static int vcodec_remove(struct platform_device *pdev)
2650 {
2651         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2652
2653         vcodec_subdev_remove(data);
2654
2655         pm_runtime_disable(data->pservice->dev);
2656
2657         return 0;
2658 }
2659
2660 static void vcodec_shutdown(struct platform_device *pdev)
2661 {
2662         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2663         struct vpu_service_info *pservice = data->pservice;
2664         struct device_node *np = pdev->dev.of_node;
2665         int val;
2666         int ret;
2667         int i;
2668
2669         dev_info(&pdev->dev, "vcodec shutdown");
2670
2671         mutex_lock(&pservice->shutdown_lock);
2672         atomic_set(&pservice->service_on, 0);
2673         mutex_unlock(&pservice->shutdown_lock);
2674
2675         ret = readx_poll_timeout(atomic_read,
2676                                  &pservice->total_running,
2677                                  val, val == 0, 20000, 200000);
2678         if (ret == -ETIMEDOUT)
2679                 dev_err(&pdev->dev, "wait total running time out\n");
2680
2681         vcodec_exit_mode(data);
2682         vpu_service_clear(data);
2683         if (of_property_read_bool(np, "subcnt")) {
2684                 for (i = 0; i < pservice->subcnt; i++) {
2685                         struct device_node *sub_np;
2686                         struct platform_device *sub_pdev;
2687
2688                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2689                         sub_pdev = of_find_device_by_node(sub_np);
2690                         vcodec_subdev_remove(platform_get_drvdata(sub_pdev));
2691                 }
2692
2693         } else {
2694                 vcodec_subdev_remove(data);
2695         }
2696
2697         pm_runtime_disable(&pdev->dev);
2698 }
2699
2700 static const struct of_device_id vcodec_service_dt_ids[] = {
2701         {
2702                 .compatible = "rockchip,vpu_service",
2703                 .data = &vpu_device_info,
2704         },
2705         {
2706                 .compatible = "rockchip,hevc_service",
2707                 .data = &hevc_device_info,
2708         },
2709         {
2710                 .compatible = "rockchip,vpu_combo",
2711                 .data = &vpu_combo_device_info,
2712         },
2713         {
2714                 .compatible = "rockchip,rkvdec",
2715                 .data = &rkvd_device_info,
2716         },
2717         {},
2718 };
2719
2720 MODULE_DEVICE_TABLE(of, vcodec_service_dt_ids);
2721
2722 static void *vcodec_get_drv_data(struct platform_device *pdev)
2723 {
2724         struct vcodec_device_info *driver_data = NULL;
2725         const struct of_device_id *match;
2726
2727         match = of_match_node(vcodec_service_dt_ids, pdev->dev.of_node);
2728         if (match)
2729                 driver_data = (struct vcodec_device_info *)match->data;
2730
2731         return driver_data;
2732 }
2733
2734 static struct platform_driver vcodec_driver = {
2735         .probe = vcodec_probe,
2736         .remove = vcodec_remove,
2737         .shutdown = vcodec_shutdown,
2738         .driver = {
2739                 .name = "rk-vcodec",
2740                 .owner = THIS_MODULE,
2741                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2742         },
2743 };
2744
2745 static void get_hw_info(struct vpu_subdev_data *data)
2746 {
2747         struct vpu_service_info *pservice = data->pservice;
2748         struct vpu_dec_config *dec = &pservice->dec_config;
2749         struct vpu_enc_config *enc = &pservice->enc_config;
2750
2751         if (of_machine_is_compatible("rockchip,rk2928") ||
2752                         of_machine_is_compatible("rockchip,rk3036") ||
2753                         of_machine_is_compatible("rockchip,rk3066") ||
2754                         of_machine_is_compatible("rockchip,rk3126") ||
2755                         of_machine_is_compatible("rockchip,rk3188"))
2756                 dec->max_dec_pic_width = 1920;
2757         else
2758                 dec->max_dec_pic_width = 4096;
2759
2760         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2761                 dec->h264_support = 3;
2762                 dec->jpeg_support = 1;
2763                 dec->mpeg4_support = 2;
2764                 dec->vc1_support = 3;
2765                 dec->mpeg2_support = 1;
2766                 dec->pp_support = 1;
2767                 dec->sorenson_support = 1;
2768                 dec->ref_buf_support = 3;
2769                 dec->vp6_support = 1;
2770                 dec->vp7_support = 1;
2771                 dec->vp8_support = 1;
2772                 dec->avs_support = 1;
2773                 dec->jpeg_ext_support = 0;
2774                 dec->custom_mpeg4_support = 1;
2775                 dec->reserve = 0;
2776                 dec->mvc_support = 1;
2777
2778                 if (!of_machine_is_compatible("rockchip,rk3036")) {
2779                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2780
2781                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2782                         enc->h264_enabled = 1;
2783                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2784                         enc->jpeg_enabled = 1;
2785                         enc->vs_enabled = (config_reg >> 24) & 1;
2786                         enc->rgb_enabled = (config_reg >> 28) & 1;
2787                         enc->reg_size = data->reg_size;
2788                         enc->reserv[0] = 0;
2789                         enc->reserv[1] = 0;
2790                 }
2791
2792                 pservice->auto_freq = true;
2793                 vpu_debug(DEBUG_EXTRA_INFO,
2794                           "vpu_service set to auto frequency mode\n");
2795                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2796
2797                 pservice->bug_dec_addr = of_machine_is_compatible
2798                         ("rockchip,rk30xx");
2799         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2800                 pservice->auto_freq = true;
2801                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2802         } else {
2803                 /* disable frequency switch in hevc.*/
2804                 pservice->auto_freq = false;
2805         }
2806 }
2807
2808 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2809 {
2810         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2811                   task->name, irq_status, task->error_mask);
2812
2813         return (task->error_mask & irq_status) ? true : false;
2814 }
2815
2816 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2817 {
2818         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2819         struct vpu_service_info *pservice = data->pservice;
2820         struct vpu_task_info *task = NULL;
2821         struct vpu_device *dev = &data->dec_dev;
2822         u32 hw_id = data->hw_info->hw_id;
2823         u32 raw_status;
2824         u32 dec_status;
2825
2826         task = &data->task_info[TASK_DEC];
2827
2828         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2829         dec_status = raw_status;
2830
2831         vpu_debug(DEBUG_TASK_INFO,
2832                   "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2833                   task->reg_irq, dec_status,
2834                   task->irq_mask, task->ready_mask, task->error_mask);
2835
2836         if (dec_status & task->irq_mask) {
2837                 time_record(task, 1);
2838                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2839                           dec_status);
2840                 if ((dec_status & 0x40001) == 0x40001) {
2841                         do {
2842                                 dec_status =
2843                                         readl_relaxed(dev->regs +
2844                                                 task->reg_irq);
2845                         } while ((dec_status & 0x40001) == 0x40001);
2846                 }
2847
2848                 if (check_irq_err(task, dec_status))
2849                         atomic_add(1, &pservice->reset_request);
2850
2851                 writel_relaxed(0, dev->regs + task->reg_irq);
2852
2853                 /* set clock gating to save power */
2854                 writel(task->gating_mask, dev->regs + task->reg_en);
2855
2856                 atomic_add(1, &dev->irq_count_codec);
2857                 time_diff(task);
2858         }
2859
2860         task = &data->task_info[TASK_PP];
2861         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2862                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2863
2864                 if (pp_status & task->irq_mask) {
2865                         time_record(task, 1);
2866                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2867                                   pp_status);
2868
2869                         if (check_irq_err(task, dec_status))
2870                                 atomic_add(1, &pservice->reset_request);
2871
2872                         /* clear pp IRQ */
2873                         writel_relaxed(pp_status & (~task->reg_irq),
2874                                        dev->regs + task->irq_mask);
2875                         atomic_add(1, &dev->irq_count_pp);
2876                         time_diff(task);
2877                 }
2878         }
2879
2880         pservice->irq_status = raw_status;
2881
2882         if (atomic_read(&dev->irq_count_pp) ||
2883             atomic_read(&dev->irq_count_codec))
2884                 return IRQ_WAKE_THREAD;
2885         else
2886                 return IRQ_NONE;
2887 }
2888
2889 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2890 {
2891         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2892         struct vpu_service_info *pservice = data->pservice;
2893         struct vpu_device *dev = &data->dec_dev;
2894
2895         mutex_lock(&pservice->lock);
2896         if (atomic_read(&dev->irq_count_codec)) {
2897                 atomic_sub(1, &dev->irq_count_codec);
2898                 if (pservice->reg_codec == NULL) {
2899                         vpu_err("error: dec isr with no task waiting\n");
2900                 } else {
2901                         reg_from_run_to_done(data, pservice->reg_codec);
2902                         /* avoid vpu timeout and can't recover problem */
2903                         if (data->mode == VCODEC_RUNNING_MODE_VPU)
2904                                 VDPU_SOFT_RESET(data->regs);
2905                 }
2906         }
2907
2908         if (atomic_read(&dev->irq_count_pp)) {
2909                 atomic_sub(1, &dev->irq_count_pp);
2910                 if (pservice->reg_pproc == NULL)
2911                         vpu_err("error: pp isr with no task waiting\n");
2912                 else
2913                         reg_from_run_to_done(data, pservice->reg_pproc);
2914         }
2915
2916         queue_work(pservice->set_workq, &data->set_work);
2917         mutex_unlock(&pservice->lock);
2918         return IRQ_HANDLED;
2919 }
2920
2921 static irqreturn_t vepu_irq(int irq, void *dev_id)
2922 {
2923         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2924         struct vpu_service_info *pservice = data->pservice;
2925         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2926         struct vpu_device *dev = &data->enc_dev;
2927         u32 irq_status;
2928
2929         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2930
2931         vpu_debug(DEBUG_TASK_INFO,
2932                   "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2933                   task->reg_irq, irq_status,
2934                   task->irq_mask, task->ready_mask, task->error_mask);
2935
2936         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2937
2938         if (likely(irq_status & task->irq_mask)) {
2939                 time_record(task, 1);
2940
2941                 if (check_irq_err(task, irq_status))
2942                         atomic_add(1, &pservice->reset_request);
2943
2944                 /* clear enc IRQ */
2945                 writel_relaxed(irq_status & (~task->irq_mask),
2946                                dev->regs + task->reg_irq);
2947
2948                 atomic_add(1, &dev->irq_count_codec);
2949                 time_diff(task);
2950         }
2951
2952         pservice->irq_status = irq_status;
2953
2954         if (atomic_read(&dev->irq_count_codec))
2955                 return IRQ_WAKE_THREAD;
2956         else
2957                 return IRQ_NONE;
2958 }
2959
2960 static irqreturn_t vepu_isr(int irq, void *dev_id)
2961 {
2962         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2963         struct vpu_service_info *pservice = data->pservice;
2964         struct vpu_device *dev = &data->enc_dev;
2965
2966         mutex_lock(&pservice->lock);
2967         if (atomic_read(&dev->irq_count_codec)) {
2968                 atomic_sub(1, &dev->irq_count_codec);
2969                 if (NULL == pservice->reg_codec)
2970                         vpu_err("error: enc isr with no task waiting\n");
2971                 else
2972                         reg_from_run_to_done(data, pservice->reg_codec);
2973         }
2974         queue_work(pservice->set_workq, &data->set_work);
2975         mutex_unlock(&pservice->lock);
2976
2977         return IRQ_HANDLED;
2978 }
2979
2980 module_platform_driver(vcodec_driver);
2981 MODULE_LICENSE("GPL v2");