drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming, chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *         Jung Zhao, jung.zhao@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/clk.h>
  21 #include <linux/compat.h>
  22 #include <linux/delay.h>
  23 #include <linux/init.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/mm.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/reset.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/wakelock.h>
  33 #include <linux/cdev.h>
  34 #include <linux/of.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/regmap.h>
  38 #include <linux/mfd/syscon.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/iopoll.h>
  43
  44 #include <linux/rockchip/cru.h>
  45 #include <linux/rockchip/pmu.h>
  46 #include <linux/rockchip/grf.h>
  47
  48 #include <linux/dma-buf.h>
  49 #include <linux/rockchip-iovmm.h>
  50
  51 #include "vcodec_hw_info.h"
  52 #include "vcodec_hw_vpu.h"
  53 #include "vcodec_hw_rkv.h"
  54 #include "vcodec_hw_vpu2.h"
  55
  56 #include "vcodec_service.h"
  57
  58 #include "vcodec_iommu_ops.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 #define MHZ                                     (1000 * 1000)
  90 #define SIZE_REG(reg)                           ((reg) * 4)
  91
  92 #define VCODEC_CLOCK_ENABLE     1
  93 #define EXTRA_INFO_MAGIC        0x4C4A46
  94
  95 static int debug;
  96 module_param(debug, int, S_IRUGO | S_IWUSR);
  97 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  98 /*
  99  * hardware information organization
 100  *
 101  * In order to support multiple hardware with different version the hardware
 102  * information is organized as follow:
 103  *
 104  * 1. First, index hardware by register size / position.
 105  *    These information is fix for each hardware and do not relate to runtime
 106  *    work flow. It only related to resource allocation.
 107  *    Descriptor: struct vpu_hw_info
 108  *
 109  * 2. Then, index hardware by runtime configuration
 110  *    These information is related to runtime setting behave including enable
 111  *    register, irq register and other key control flag
 112  *    Descriptor: struct vpu_task_info
 113  *
 114  * 3. Final, on iommu case the fd translation is required
 115  *    Descriptor: struct vpu_trans_info
 116  */
 117
 118 enum VPU_FREQ {
 119         VPU_FREQ_200M,
 120         VPU_FREQ_266M,
 121         VPU_FREQ_300M,
 122         VPU_FREQ_400M,
 123         VPU_FREQ_500M,
 124         VPU_FREQ_600M,
 125         VPU_FREQ_DEFAULT,
 126         VPU_FREQ_BUT,
 127 };
 128
 129 struct extra_info_elem {
 130         u32 index;
 131         u32 offset;
 132 };
 133
 134
 135 struct extra_info_for_iommu {
 136         u32 magic;
 137         u32 cnt;
 138         struct extra_info_elem elem[20];
 139 };
 140
 141 static const struct vcodec_info vcodec_info_set[] = {
 142         {
 143                 .hw_id          = VPU_ID_8270,
 144                 .hw_info        = &hw_vpu_8270,
 145                 .task_info      = task_vpu,
 146                 .trans_info     = trans_vpu,
 147         },
 148         {
 149                 .hw_id          = VPU_ID_4831,
 150                 .hw_info        = &hw_vpu_4831,
 151                 .task_info      = task_vpu,
 152                 .trans_info     = trans_vpu,
 153         },
 154         {
 155                 .hw_id          = VPU_DEC_ID_9190,
 156                 .hw_info        = &hw_vpu_9190,
 157                 .task_info      = task_vpu,
 158                 .trans_info     = trans_vpu,
 159         },
 160         {
 161                 .hw_id          = HEVC_ID,
 162                 .hw_info        = &hw_rkhevc,
 163                 .task_info      = task_rkv,
 164                 .trans_info     = trans_rkv,
 165         },
 166         {
 167                 .hw_id          = RKV_DEC_ID,
 168                 .hw_info        = &hw_rkvdec,
 169                 .task_info      = task_rkv,
 170                 .trans_info     = trans_rkv,
 171         },
 172         {
 173                 .hw_id          = VPU2_ID,
 174                 .hw_info        = &hw_vpu2,
 175                 .task_info      = task_vpu2,
 176                 .trans_info     = trans_vpu2,
 177         },
 178 };
 179
 180 /* Both VPU1 and VPU2 */
 181 static const struct vcodec_device_info vpu_device_info = {
 182         .device_type = VCODEC_DEVICE_TYPE_VPUX,
 183         .name = "vpu-service",
 184 };
 185
 186 static const struct vcodec_device_info vpu_combo_device_info = {
 187         .device_type = VCODEC_DEVICE_TYPE_VPUC,
 188         .name = "vpu-combo",
 189 };
 190
 191 static const struct vcodec_device_info hevc_device_info = {
 192         .device_type = VCODEC_DEVICE_TYPE_HEVC,
 193         .name = "hevc-service",
 194 };
 195
 196 static const struct vcodec_device_info rkvd_device_info = {
 197         .device_type = VCODEC_DEVICE_TYPE_RKVD,
 198         .name = "rkvdec",
 199 };
 200
 201 #define DEBUG
 202 #ifdef DEBUG
 203 #define vpu_debug_func(type, fmt, args...)                      \
 204         do {                                                    \
 205                 if (unlikely(debug & type)) {                   \
 206                         pr_info("%s:%d: " fmt,                  \
 207                                  __func__, __LINE__, ##args);   \
 208                 }                                               \
 209         } while (0)
 210 #define vpu_debug(type, fmt, args...)                           \
 211         do {                                                    \
 212                 if (unlikely(debug & type)) {                   \
 213                         pr_info(fmt, ##args);                   \
 214                 }                                               \
 215         } while (0)
 216 #else
 217 #define vpu_debug_func(level, fmt, args...)
 218 #define vpu_debug(level, fmt, args...)
 219 #endif
 220
 221 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 222 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 223
 224 #define vpu_err(fmt, args...)                           \
 225                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 226
 227 enum VPU_DEC_FMT {
 228         VPU_DEC_FMT_H264,
 229         VPU_DEC_FMT_MPEG4,
 230         VPU_DEC_FMT_H263,
 231         VPU_DEC_FMT_JPEG,
 232         VPU_DEC_FMT_VC1,
 233         VPU_DEC_FMT_MPEG2,
 234         VPU_DEC_FMT_MPEG1,
 235         VPU_DEC_FMT_VP6,
 236         VPU_DEC_FMT_RESERV0,
 237         VPU_DEC_FMT_VP7,
 238         VPU_DEC_FMT_VP8,
 239         VPU_DEC_FMT_AVS,
 240         VPU_DEC_FMT_RES
 241 };
 242
 243 /**
 244  * struct for process session which connect to vpu
 245  *
 246  * @author ChenHengming (2011-5-3)
 247  */
 248 struct vpu_session {
 249         enum VPU_CLIENT_TYPE type;
 250         /* a linked list of data so we can access them for debugging */
 251         struct list_head list_session;
 252         /* a linked list of register data waiting for process */
 253         struct list_head waiting;
 254         /* a linked list of register data in processing */
 255         struct list_head running;
 256         /* a linked list of register data processed */
 257         struct list_head done;
 258         wait_queue_head_t wait;
 259         pid_t pid;
 260         atomic_t task_running;
 261 };
 262
 263 /**
 264  * struct for process register set
 265  *
 266  * @author ChenHengming (2011-5-4)
 267  */
 268 struct vpu_reg {
 269         enum VPU_CLIENT_TYPE type;
 270         enum VPU_FREQ freq;
 271         struct vpu_session *session;
 272         struct vpu_subdev_data *data;
 273         struct vpu_task_info *task;
 274         const struct vpu_trans_info *trans;
 275
 276         /* link to vpu service session */
 277         struct list_head session_link;
 278         /* link to register set list */
 279         struct list_head status_link;
 280
 281         unsigned long size;
 282         struct list_head mem_region_list;
 283         u32 dec_base;
 284         u32 *reg;
 285 };
 286
 287 struct vpu_device {
 288         atomic_t irq_count_codec;
 289         atomic_t irq_count_pp;
 290         unsigned int iosize;
 291         u32 *regs;
 292 };
 293
 294 enum vcodec_device_id {
 295         VCODEC_DEVICE_ID_VPU,
 296         VCODEC_DEVICE_ID_HEVC,
 297         VCODEC_DEVICE_ID_COMBO,
 298         VCODEC_DEVICE_ID_RKVDEC,
 299         VCODEC_DEVICE_ID_BUTT
 300 };
 301
 302 enum VCODEC_RUNNING_MODE {
 303         VCODEC_RUNNING_MODE_NONE = -1,
 304         VCODEC_RUNNING_MODE_VPU,
 305         VCODEC_RUNNING_MODE_HEVC,
 306         VCODEC_RUNNING_MODE_RKVDEC
 307 };
 308
 309 struct vcodec_mem_region {
 310         struct list_head srv_lnk;
 311         struct list_head reg_lnk;
 312         struct list_head session_lnk;
 313         unsigned long iova;     /* virtual address for iommu */
 314         unsigned long len;
 315         u32 reg_idx;
 316         int hdl;
 317 };
 318
 319 enum vpu_ctx_state {
 320         MMU_ACTIVATED   = BIT(0)
 321 };
 322
 323 struct vpu_subdev_data {
 324         struct cdev cdev;
 325         dev_t dev_t;
 326         struct class *cls;
 327         struct device *child_dev;
 328
 329         int irq_enc;
 330         int irq_dec;
 331         struct vpu_service_info *pservice;
 332
 333         u32 *regs;
 334         enum VCODEC_RUNNING_MODE mode;
 335         struct list_head lnk_service;
 336
 337         struct device *dev;
 338
 339         struct vpu_device enc_dev;
 340         struct vpu_device dec_dev;
 341
 342         enum VPU_HW_ID hw_id;
 343         struct vpu_hw_info *hw_info;
 344         struct vpu_task_info *task_info;
 345         const struct vpu_trans_info *trans_info;
 346
 347         u32 reg_size;
 348         unsigned long state;
 349
 350 #ifdef CONFIG_DEBUG_FS
 351         struct dentry *debugfs_dir;
 352         struct dentry *debugfs_file_regs;
 353 #endif
 354
 355         struct device *mmu_dev;
 356         struct vcodec_iommu_info *iommu_info;
 357 };
 358
 359 struct vpu_service_info {
 360         struct wake_lock wake_lock;
 361         struct delayed_work power_off_work;
 362         ktime_t last; /* record previous power-on time */
 363         /* vpu service structure global lock */
 364         struct mutex lock;
 365         /* link to link_reg in struct vpu_reg */
 366         struct list_head waiting;
 367         /* link to link_reg in struct vpu_reg */
 368         struct list_head running;
 369         /* link to link_reg in struct vpu_reg */
 370         struct list_head done;
 371         /* link to list_session in struct vpu_session */
 372         struct list_head session;
 373         atomic_t total_running;
 374         atomic_t enabled;
 375         atomic_t power_on_cnt;
 376         atomic_t power_off_cnt;
 377         atomic_t service_on;
 378         struct mutex shutdown_lock;
 379         struct vpu_reg *reg_codec;
 380         struct vpu_reg *reg_pproc;
 381         struct vpu_reg *reg_resev;
 382         struct vpu_dec_config dec_config;
 383         struct vpu_enc_config enc_config;
 384
 385         bool auto_freq;
 386         bool bug_dec_addr;
 387         atomic_t freq_status;
 388
 389         struct clk *aclk_vcodec;
 390         struct clk *hclk_vcodec;
 391         struct clk *clk_core;
 392         struct clk *clk_cabac;
 393         struct clk *pd_video;
 394
 395 #ifdef CONFIG_RESET_CONTROLLER
 396         struct reset_control *rst_a;
 397         struct reset_control *rst_h;
 398         struct reset_control *rst_v;
 399 #endif
 400         struct device *dev;
 401
 402         u32 irq_status;
 403         atomic_t reset_request;
 404         struct list_head mem_region_list;
 405
 406         enum vcodec_device_id dev_id;
 407
 408         enum VCODEC_RUNNING_MODE curr_mode;
 409         u32 prev_mode;
 410
 411         struct delayed_work simulate_work;
 412
 413         u32 mode_bit;
 414         u32 mode_ctrl;
 415         u32 *reg_base;
 416         u32 ioaddr;
 417         struct regmap *grf;
 418         u32 *grf_base;
 419
 420         char *name;
 421
 422         u32 subcnt;
 423         struct list_head subdev_list;
 424
 425         u32 alloc_type;
 426 };
 427
 428 struct vpu_request {
 429         u32 *req;
 430         u32 size;
 431 };
 432
 433 #ifdef CONFIG_COMPAT
 434 struct compat_vpu_request {
 435         compat_uptr_t req;
 436         u32 size;
 437 };
 438 #endif
 439
 440 #define VDPU_SOFT_RESET_REG     101
 441 #define VDPU_CLEAN_CACHE_REG    516
 442 #define VEPU_CLEAN_CACHE_REG    772
 443 #define HEVC_CLEAN_CACHE_REG    260
 444
 445 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 446
 447 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 448 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 449 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 450 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 451
 452 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 453 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 454
 455 static void *vcodec_get_drv_data(struct platform_device *pdev);
 456
 457 static void vpu_service_power_on(struct vpu_subdev_data *data,
 458                                  struct vpu_service_info *pservice);
 459
 460 static void time_record(struct vpu_task_info *task, int is_end)
 461 {
 462         if (unlikely(debug & DEBUG_TIMING) && task)
 463                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 464 }
 465
 466 static void time_diff(struct vpu_task_info *task)
 467 {
 468         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 469                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 470                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 471 }
 472
 473 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 474 {
 475         int bits;
 476         u32 raw = 0;
 477         struct vpu_service_info *pservice = data->pservice;
 478         struct vpu_subdev_data *subdata, *n;
 479
 480         if (pservice->subcnt < 2)
 481                 return;
 482
 483         if (pservice->curr_mode == data->mode)
 484                 return;
 485
 486         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 487         list_for_each_entry_safe(subdata, n,
 488                                  &pservice->subdev_list, lnk_service) {
 489                 if (data != subdata && subdata->mmu_dev &&
 490                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 491                         clear_bit(MMU_ACTIVATED, &subdata->state);
 492                 }
 493         }
 494         bits = 1 << pservice->mode_bit;
 495 #ifdef CONFIG_MFD_SYSCON
 496         if (pservice->grf) {
 497                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 498
 499                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 500                         regmap_write(pservice->grf, pservice->mode_ctrl,
 501                                      raw | bits | (bits << 16));
 502                 else
 503                         regmap_write(pservice->grf, pservice->mode_ctrl,
 504                                      (raw & (~bits)) | (bits << 16));
 505         } else if (pservice->grf_base) {
 506                 u32 *grf_base = pservice->grf_base;
 507
 508                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 509                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 510                         writel_relaxed(raw | bits | (bits << 16),
 511                                        grf_base + pservice->mode_ctrl / 4);
 512                 else
 513                         writel_relaxed((raw & (~bits)) | (bits << 16),
 514                                        grf_base + pservice->mode_ctrl / 4);
 515         } else {
 516                 vpu_err("no grf resource define, switch decoder failed\n");
 517                 return;
 518         }
 519 #else
 520         if (pservice->grf_base) {
 521                 u32 *grf_base = pservice->grf_base;
 522
 523                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 524                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 525                         writel_relaxed(raw | bits | (bits << 16),
 526                                        grf_base + pservice->mode_ctrl / 4);
 527                 else
 528                         writel_relaxed((raw & (~bits)) | (bits << 16),
 529                                        grf_base + pservice->mode_ctrl / 4);
 530         } else {
 531                 vpu_err("no grf resource define, switch decoder failed\n");
 532                 return;
 533         }
 534 #endif
 535         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 536                 set_bit(MMU_ACTIVATED, &data->state);
 537                 if (!atomic_read(&pservice->enabled))
 538                         /* FIXME BUG_ON should not be used in mass produce */
 539                         BUG_ON(!atomic_read(&pservice->enabled));
 540         }
 541
 542         pservice->prev_mode = pservice->curr_mode;
 543         pservice->curr_mode = data->mode;
 544 }
 545
 546 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 547 {
 548         /*
 549          * In case of VPU Combo, it require HW switch its running mode
 550          * before the other HW component start work. set current HW running
 551          * mode to none, can ensure HW switch to its reqired mode properly.
 552          */
 553         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 554 }
 555
 556 static int vpu_get_clk(struct vpu_service_info *pservice)
 557 {
 558 #if VCODEC_CLOCK_ENABLE
 559         struct device *dev = pservice->dev;
 560
 561         switch (pservice->dev_id) {
 562         case VCODEC_DEVICE_ID_HEVC:
 563                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 564                 if (IS_ERR(pservice->pd_video)) {
 565                         pservice->pd_video = NULL;
 566                         dev_info(dev, "failed on clk_get pd_hevc\n");
 567                 }
 568         case VCODEC_DEVICE_ID_COMBO:
 569         case VCODEC_DEVICE_ID_RKVDEC:
 570                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 571                 if (IS_ERR(pservice->clk_cabac)) {
 572                         dev_err(dev, "failed on clk_get clk_cabac\n");
 573                         pservice->clk_cabac = NULL;
 574                 }
 575                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 576                 if (IS_ERR(pservice->clk_core)) {
 577                         dev_err(dev, "failed on clk_get clk_core\n");
 578                         pservice->clk_core = NULL;
 579                         return -1;
 580                 }
 581         case VCODEC_DEVICE_ID_VPU:
 582                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 583                 if (IS_ERR(pservice->aclk_vcodec)) {
 584                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 585                         pservice->aclk_vcodec = NULL;
 586                         return -1;
 587                 }
 588
 589                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 590                 if (IS_ERR(pservice->hclk_vcodec)) {
 591                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 592                         pservice->hclk_vcodec = NULL;
 593                         return -1;
 594                 }
 595                 if (pservice->pd_video == NULL) {
 596                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 597                         if (IS_ERR(pservice->pd_video)) {
 598                                 pservice->pd_video = NULL;
 599                                 dev_info(dev, "do not have pd_video\n");
 600                         }
 601                 }
 602                 break;
 603         default:
 604                 break;
 605         }
 606
 607         return 0;
 608 #else
 609         return 0;
 610 #endif
 611 }
 612
 613 static void _vpu_reset(struct vpu_subdev_data *data)
 614 {
 615         struct vpu_service_info *pservice = data->pservice;
 616         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 617
 618         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 619                 type = IDLE_REQ_HEVC;
 620
 621         dev_info(pservice->dev, "resetting...\n");
 622         WARN_ON(pservice->reg_codec != NULL);
 623         WARN_ON(pservice->reg_pproc != NULL);
 624         WARN_ON(pservice->reg_resev != NULL);
 625         pservice->reg_codec = NULL;
 626         pservice->reg_pproc = NULL;
 627         pservice->reg_resev = NULL;
 628
 629 #ifdef CONFIG_RESET_CONTROLLER
 630         dev_info(pservice->dev, "for 3288/3368...");
 631         if (of_machine_is_compatible("rockchip,rk3288"))
 632                 rockchip_pmu_idle_request(pservice->dev, true);
 633         if (pservice->rst_a && pservice->rst_h) {
 634                 dev_info(pservice->dev, "vpu reset in\n");
 635
 636                 if (pservice->rst_v)
 637                         reset_control_assert(pservice->rst_v);
 638                 reset_control_assert(pservice->rst_a);
 639                 reset_control_assert(pservice->rst_h);
 640                 udelay(5);
 641
 642                 reset_control_deassert(pservice->rst_h);
 643                 reset_control_deassert(pservice->rst_a);
 644                 if (pservice->rst_v)
 645                         reset_control_deassert(pservice->rst_v);
 646         } else if (pservice->rst_v) {
 647                 dev_info(pservice->dev, "hevc reset in\n");
 648                 reset_control_assert(pservice->rst_v);
 649                 udelay(5);
 650
 651                 reset_control_deassert(pservice->rst_v);
 652         }
 653         if (of_machine_is_compatible("rockchip,rk3288"))
 654                 rockchip_pmu_idle_request(pservice->dev, false);
 655 #endif
 656 }
 657
 658 static void vpu_reset(struct vpu_subdev_data *data)
 659 {
 660         struct vpu_service_info *pservice = data->pservice;
 661
 662         _vpu_reset(data);
 663         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 664                 if (atomic_read(&pservice->enabled)) {
 665                         /* Need to reset iommu */
 666                         vcodec_iommu_detach(data->iommu_info);
 667                         vcodec_iommu_attach(data->iommu_info);
 668                 } else {
 669                         /* FIXME BUG_ON should not be used in mass produce */
 670                         BUG_ON(!atomic_read(&pservice->enabled));
 671                 }
 672         }
 673
 674         atomic_set(&pservice->reset_request, 0);
 675         dev_info(pservice->dev, "reset done\n");
 676 }
 677
 678 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 679 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 680                                       struct vpu_session *session)
 681 {
 682         struct vpu_reg *reg, *n;
 683
 684         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 685                 reg_deinit(data, reg);
 686         }
 687         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 688                 reg_deinit(data, reg);
 689         }
 690         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 691                 reg_deinit(data, reg);
 692         }
 693 }
 694
 695 static void vpu_service_clear(struct vpu_subdev_data *data)
 696 {
 697         struct vpu_reg *reg, *n;
 698         struct vpu_session *session, *s;
 699         struct vpu_service_info *pservice = data->pservice;
 700
 701         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 702                 reg_deinit(reg->data, reg);
 703         }
 704
 705         /* wake up session wait event to prevent the timeout hw reset
 706          * during reboot procedure.
 707          */
 708         list_for_each_entry_safe(session, s,
 709                                  &pservice->session, list_session)
 710                 wake_up(&session->wait);
 711 }
 712
 713 static void vpu_service_dump(struct vpu_service_info *pservice)
 714 {
 715 }
 716
 717
 718 static void vpu_service_power_off(struct vpu_service_info *pservice)
 719 {
 720         int total_running;
 721         struct vpu_subdev_data *data = NULL, *n;
 722         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 723
 724         if (!ret)
 725                 return;
 726
 727         total_running = atomic_read(&pservice->total_running);
 728         if (total_running) {
 729                 pr_alert("alert: power off when %d task running!!\n",
 730                          total_running);
 731                 mdelay(50);
 732                 pr_alert("alert: delay 50 ms for running task\n");
 733                 vpu_service_dump(pservice);
 734         }
 735
 736         dev_dbg(pservice->dev, "power off...\n");
 737
 738         udelay(5);
 739
 740         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 741                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 742                         clear_bit(MMU_ACTIVATED, &data->state);
 743                         vcodec_iommu_detach(data->iommu_info);
 744                 }
 745         }
 746         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 747         pm_runtime_put(pservice->dev);
 748 #if VCODEC_CLOCK_ENABLE
 749                 if (pservice->pd_video)
 750                         clk_disable_unprepare(pservice->pd_video);
 751                 if (pservice->hclk_vcodec)
 752                         clk_disable_unprepare(pservice->hclk_vcodec);
 753                 if (pservice->aclk_vcodec)
 754                         clk_disable_unprepare(pservice->aclk_vcodec);
 755                 if (pservice->clk_core)
 756                         clk_disable_unprepare(pservice->clk_core);
 757                 if (pservice->clk_cabac)
 758                         clk_disable_unprepare(pservice->clk_cabac);
 759 #endif
 760
 761         atomic_add(1, &pservice->power_off_cnt);
 762         wake_unlock(&pservice->wake_lock);
 763         dev_dbg(pservice->dev, "power off done\n");
 764 }
 765
 766 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 767 {
 768         queue_delayed_work(system_wq, &pservice->power_off_work,
 769                            VPU_POWER_OFF_DELAY);
 770 }
 771
 772 static void vpu_power_off_work(struct work_struct *work_s)
 773 {
 774         struct delayed_work *dlwork = container_of(work_s,
 775                         struct delayed_work, work);
 776         struct vpu_service_info *pservice = container_of(dlwork,
 777                         struct vpu_service_info, power_off_work);
 778
 779         if (mutex_trylock(&pservice->lock)) {
 780                 vpu_service_power_off(pservice);
 781                 mutex_unlock(&pservice->lock);
 782         } else {
 783                 /* Come back later if the device is busy... */
 784                 vpu_queue_power_off_work(pservice);
 785         }
 786 }
 787
 788 static void vpu_service_power_on(struct vpu_subdev_data *data,
 789                                  struct vpu_service_info *pservice)
 790 {
 791         int ret;
 792         ktime_t now = ktime_get();
 793
 794         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC ||
 795             atomic_read(&pservice->power_on_cnt)) {
 796                 /* NSEC_PER_SEC */
 797                 cancel_delayed_work_sync(&pservice->power_off_work);
 798                 vpu_queue_power_off_work(pservice);
 799                 pservice->last = now;
 800         }
 801         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 802         if (!ret) {
 803                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 804                         set_bit(MMU_ACTIVATED, &data->state);
 805                         vcodec_iommu_attach(data->iommu_info);
 806                 }
 807                 return;
 808         }
 809
 810         dev_dbg(pservice->dev, "power on\n");
 811
 812 #define BIT_VCODEC_CLK_SEL      (1<<10)
 813         if (of_machine_is_compatible("rockchip,rk3126"))
 814                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 815                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 816                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 817
 818 #if VCODEC_CLOCK_ENABLE
 819         if (pservice->aclk_vcodec)
 820                 clk_prepare_enable(pservice->aclk_vcodec);
 821         if (pservice->hclk_vcodec)
 822                 clk_prepare_enable(pservice->hclk_vcodec);
 823         if (pservice->clk_core)
 824                 clk_prepare_enable(pservice->clk_core);
 825         if (pservice->clk_cabac)
 826                 clk_prepare_enable(pservice->clk_cabac);
 827         if (pservice->pd_video)
 828                 clk_prepare_enable(pservice->pd_video);
 829 #endif
 830         pm_runtime_get_sync(pservice->dev);
 831
 832         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 833                 set_bit(MMU_ACTIVATED, &data->state);
 834                 if (atomic_read(&pservice->enabled))
 835                         vcodec_iommu_attach(data->iommu_info);
 836                 else
 837                         /*
 838                          * FIXME BUG_ON should not be used in mass
 839                          * produce.
 840                          */
 841                         BUG_ON(!atomic_read(&pservice->enabled));
 842         }
 843
 844         udelay(5);
 845         atomic_add(1, &pservice->power_on_cnt);
 846         wake_lock(&pservice->wake_lock);
 847 }
 848
 849 static inline bool reg_check_interlace(struct vpu_reg *reg)
 850 {
 851         u32 type = (reg->reg[3] & (1 << 23));
 852
 853         return (type > 0);
 854 }
 855
 856 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 857 {
 858         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 859
 860         return type;
 861 }
 862
 863 static inline int reg_probe_width(struct vpu_reg *reg)
 864 {
 865         int width_in_mb = reg->reg[4] >> 23;
 866
 867         return width_in_mb * 16;
 868 }
 869
 870 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 871 {
 872         int y_virstride = reg->reg[8];
 873
 874         return y_virstride;
 875 }
 876
 877 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 878                 struct vpu_session *session,
 879                 struct vpu_reg *reg,
 880                 int fd)
 881 {
 882         int hdl;
 883         int ret = 0;
 884         struct vcodec_mem_region *mem_region;
 885
 886         hdl = vcodec_iommu_import(data->iommu_info, session, fd);
 887         if (hdl < 0)
 888                 return hdl;
 889
 890         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 891         if (mem_region == NULL) {
 892                 vpu_err("allocate memory for iommu memory region failed\n");
 893                 vcodec_iommu_free(data->iommu_info, session, hdl);
 894                 return -ENOMEM;
 895         }
 896
 897         mem_region->hdl = hdl;
 898         ret = vcodec_iommu_map_iommu(data->iommu_info, session, mem_region->hdl,
 899                                      &mem_region->iova, &mem_region->len);
 900         if (ret < 0) {
 901                 vpu_err("fd %d ion map iommu failed\n", fd);
 902                 kfree(mem_region);
 903                 vcodec_iommu_free(data->iommu_info, session, hdl);
 904
 905                 return -EFAULT;
 906         }
 907         INIT_LIST_HEAD(&mem_region->reg_lnk);
 908         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 909         return mem_region->iova;
 910 }
 911
 912 /*
 913  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 914  * it by pps id in video stream data.
 915  *
 916  * So we need to translate the address in iommu case. The address data is also
 917  * 10bit fd + 22bit offset mode.
 918  * Because userspace decoder do not give the pps id in the register file sets
 919  * kernel driver need to translate each scaling list address in pps buffer which
 920  * means 256 pps for H.264, 64 pps for H.265.
 921  *
 922  * In order to optimize the performance kernel driver ask userspace decoder to
 923  * set all scaling list address in pps buffer to the same one which will be used
 924  * on current decoding task. Then kernel driver can only translate the first
 925  * address then copy it all pps buffer.
 926  */
 927 static int fill_scaling_list_addr_in_pps(
 928                 struct vpu_subdev_data *data,
 929                 struct vpu_reg *reg,
 930                 char *pps,
 931                 int pps_info_count,
 932                 int pps_info_size,
 933                 int scaling_list_addr_offset)
 934 {
 935         int base = scaling_list_addr_offset;
 936         int scaling_fd = 0;
 937         u32 scaling_offset;
 938
 939         scaling_offset  = (u32)pps[base + 0];
 940         scaling_offset += (u32)pps[base + 1] << 8;
 941         scaling_offset += (u32)pps[base + 2] << 16;
 942         scaling_offset += (u32)pps[base + 3] << 24;
 943
 944         scaling_fd = scaling_offset & 0x3ff;
 945         scaling_offset = scaling_offset >> 10;
 946
 947         if (scaling_fd > 0) {
 948                 int i = 0;
 949                 u32 tmp = vcodec_fd_to_iova(data, reg->session, reg,
 950                                             scaling_fd);
 951
 952                 if (IS_ERR_VALUE(tmp))
 953                         return -1;
 954                 tmp += scaling_offset;
 955
 956                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 957                         pps[base + 0] = (tmp >>  0) & 0xff;
 958                         pps[base + 1] = (tmp >>  8) & 0xff;
 959                         pps[base + 2] = (tmp >> 16) & 0xff;
 960                         pps[base + 3] = (tmp >> 24) & 0xff;
 961                 }
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data,
 968                                 struct vpu_session *session,
 969                                 const u8 *tbl,
 970                                 int size, struct vpu_reg *reg,
 971                                 struct extra_info_for_iommu *ext_inf)
 972 {
 973         struct vpu_service_info *pservice = data->pservice;
 974         struct vpu_task_info *task = reg->task;
 975         enum FORMAT_TYPE type;
 976         int hdl;
 977         int ret = 0;
 978         struct vcodec_mem_region *mem_region;
 979         int i;
 980         int offset = 0;
 981
 982         if (tbl == NULL || size <= 0) {
 983                 dev_err(pservice->dev, "input arguments invalidate\n");
 984                 return -EINVAL;
 985         }
 986
 987         if (task->get_fmt)
 988                 type = task->get_fmt(reg->reg);
 989         else {
 990                 dev_err(pservice->dev, "invalid task with NULL get_fmt\n");
 991                 return -EINVAL;
 992         }
 993
 994         for (i = 0; i < size; i++) {
 995                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
 996
 997                 /* if userspace do not set the fd at this register, skip */
 998                 if (usr_fd == 0)
 999                         continue;
1000
1001                 /*
1002                  * for avoiding cache sync issue, we need to map/unmap
1003                  * input buffer every time. FIX ME, if it is unnecessary
1004                  */
1005                 if (task->reg_rlc == tbl[i])
1006                         vcodec_iommu_free_fd(data->iommu_info, session, usr_fd);
1007                 /*
1008                  * special offset scale case
1009                  *
1010                  * This translation is for fd + offset translation.
1011                  * One register has 32bits. We need to transfer both buffer file
1012                  * handle and the start address offset so we packet file handle
1013                  * and offset together using below format.
1014                  *
1015                  *  0~9  bit for buffer file handle range 0 ~ 1023
1016                  * 10~31 bit for offset range 0 ~ 4M
1017                  *
1018                  * But on 4K case the offset can be larger the 4M
1019                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1020                  * But MPEG4 will use the same register for colmv and it do not
1021                  * need scale.
1022                  *
1023                  * RKVdec do not have this issue.
1024                  */
1025                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1026                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1027                         offset = reg->reg[tbl[i]] >> 10 << 4;
1028                 else
1029                         offset = reg->reg[tbl[i]] >> 10;
1030
1031                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d i %d\n",
1032                           tbl[i], usr_fd, offset, i);
1033
1034                 hdl = vcodec_iommu_import(data->iommu_info, session, usr_fd);
1035
1036                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1037                         int pps_info_offset;
1038                         int pps_info_count;
1039                         int pps_info_size;
1040                         int scaling_list_addr_offset;
1041
1042                         switch (type) {
1043                         case FMT_H264D: {
1044                                 pps_info_offset = offset;
1045                                 pps_info_count = 256;
1046                                 pps_info_size = 32;
1047                                 scaling_list_addr_offset = 23;
1048                         } break;
1049                         case FMT_H265D: {
1050                                 pps_info_offset = 0;
1051                                 pps_info_count = 64;
1052                                 pps_info_size = 80;
1053                                 scaling_list_addr_offset = 74;
1054                         } break;
1055                         default: {
1056                                 pps_info_offset = 0;
1057                                 pps_info_count = 0;
1058                                 pps_info_size = 0;
1059                                 scaling_list_addr_offset = 0;
1060                         } break;
1061                         }
1062
1063                         vpu_debug(DEBUG_PPS_FILL,
1064                                   "scaling list filling parameter:\n");
1065                         vpu_debug(DEBUG_PPS_FILL,
1066                                   "pps_info_offset %d\n", pps_info_offset);
1067                         vpu_debug(DEBUG_PPS_FILL,
1068                                   "pps_info_count  %d\n", pps_info_count);
1069                         vpu_debug(DEBUG_PPS_FILL,
1070                                   "pps_info_size   %d\n", pps_info_size);
1071                         vpu_debug(DEBUG_PPS_FILL,
1072                                   "scaling_list_addr_offset %d\n",
1073                                   scaling_list_addr_offset);
1074
1075                         if (pps_info_count) {
1076                                 u8 *pps;
1077
1078                                 mutex_lock(&pservice->lock);
1079
1080                                 pps = vcodec_iommu_map_kernel
1081                                         (data->iommu_info, session, hdl);
1082
1083                                 vpu_debug(DEBUG_PPS_FILL,
1084                                           "scaling list setting pps %p\n", pps);
1085                                 pps += pps_info_offset;
1086
1087                                 fill_scaling_list_addr_in_pps
1088                                         (data, reg, pps, pps_info_count,
1089                                          pps_info_size,
1090                                          scaling_list_addr_offset);
1091
1092                                 vcodec_iommu_unmap_kernel
1093                                         (data->iommu_info, session, hdl);
1094                                 mutex_unlock(&pservice->lock);
1095                         }
1096                 }
1097
1098                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1099
1100                 if (!mem_region) {
1101                         vcodec_iommu_free(data->iommu_info, session, hdl);
1102                         return -ENOMEM;
1103                 }
1104
1105                 mem_region->hdl = hdl;
1106                 mem_region->reg_idx = tbl[i];
1107
1108                 ret = vcodec_iommu_map_iommu(data->iommu_info, session,
1109                                              mem_region->hdl, &mem_region->iova,
1110                                              &mem_region->len);
1111                 if (ret < 0) {
1112                         dev_err(pservice->dev,
1113                                 "reg %d fd %d ion map iommu failed\n",
1114                                 tbl[i], usr_fd);
1115                         kfree(mem_region);
1116                         vcodec_iommu_free(data->iommu_info, session, hdl);
1117                         return ret;
1118                 }
1119
1120                 /*
1121                  * special for vpu dec num 12: record decoded length
1122                  * hacking for decoded length
1123                  * NOTE: not a perfect fix, the fd is not recorded
1124                  */
1125                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1126                         reg->dec_base = mem_region->iova + offset;
1127                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1128                                   reg->dec_base);
1129                 }
1130
1131                 reg->reg[tbl[i]] = mem_region->iova + offset;
1132                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1133                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1134         }
1135
1136         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1137                 for (i = 0; i < ext_inf->cnt; i++) {
1138                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1139                                   ext_inf->elem[i].index,
1140                                   ext_inf->elem[i].offset);
1141                         reg->reg[ext_inf->elem[i].index] +=
1142                                 ext_inf->elem[i].offset;
1143                 }
1144         }
1145
1146         return 0;
1147 }
1148
1149 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1150                                         struct vpu_session *session,
1151                                         struct vpu_reg *reg,
1152                                         struct extra_info_for_iommu *ext_inf)
1153 {
1154         struct vpu_service_info *pservice = data->pservice;
1155         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1156
1157         if (type < FMT_TYPE_BUTT) {
1158                 const struct vpu_trans_info *info = &reg->trans[type];
1159                 const u8 *tbl = info->table;
1160                 int size = info->count;
1161
1162                 return vcodec_bufid_to_iova(data, session, tbl, size, reg,
1163                                             ext_inf);
1164         }
1165
1166         dev_err(pservice->dev, "found invalid format type!\n");
1167         return -EINVAL;
1168 }
1169
1170 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1171 {
1172
1173         if (!of_machine_is_compatible("rockchip,rk2928g")) {
1174                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1175                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1176                                 if (reg_probe_width(reg) > 3200) {
1177                                         /*raise frequency for 4k avc.*/
1178                                         reg->freq = VPU_FREQ_600M;
1179                                 }
1180                         } else {
1181                                 if (reg_check_interlace(reg))
1182                                         reg->freq = VPU_FREQ_400M;
1183                         }
1184                 }
1185                 if (data->hw_id == HEVC_ID) {
1186                         if (reg_probe_hevc_y_stride(reg) > 60000)
1187                                 reg->freq = VPU_FREQ_400M;
1188                 }
1189                 if (reg->type == VPU_PP)
1190                         reg->freq = VPU_FREQ_400M;
1191         }
1192 }
1193
1194 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1195                                 struct vpu_session *session,
1196                                 void __user *src, u32 size)
1197 {
1198         struct vpu_service_info *pservice = data->pservice;
1199         int extra_size = 0;
1200         struct extra_info_for_iommu extra_info;
1201         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1202                                       GFP_KERNEL);
1203
1204         vpu_debug_enter();
1205
1206         if (!reg) {
1207                 vpu_err("error: kzalloc failed\n");
1208                 return NULL;
1209         }
1210
1211         if (size > data->reg_size) {
1212                 extra_size = size - data->reg_size;
1213                 size = data->reg_size;
1214         }
1215         reg->session = session;
1216         reg->data = data;
1217         reg->type = session->type;
1218         reg->size = size;
1219         reg->freq = VPU_FREQ_DEFAULT;
1220         reg->task = &data->task_info[session->type];
1221         reg->trans = data->trans_info;
1222         reg->reg = (u32 *)&reg[1];
1223         INIT_LIST_HEAD(&reg->session_link);
1224         INIT_LIST_HEAD(&reg->status_link);
1225
1226         INIT_LIST_HEAD(&reg->mem_region_list);
1227
1228         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1229                 vpu_err("error: copy_from_user failed\n");
1230                 kfree(reg);
1231                 return NULL;
1232         }
1233
1234         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1235                 vpu_err("error: copy_from_user failed\n");
1236                 kfree(reg);
1237                 return NULL;
1238         }
1239
1240         if (vcodec_reg_address_translate(data, session, reg, &extra_info) < 0) {
1241                 int i = 0;
1242
1243                 vpu_err("error: translate reg address failed, dumping regs\n");
1244                 for (i = 0; i < size >> 2; i++)
1245                         dev_err(pservice->dev, "reg[%02d]: %08x\n",
1246                                 i, *((u32 *)src + i));
1247
1248                 kfree(reg);
1249                 return NULL;
1250         }
1251
1252         mutex_lock(&pservice->lock);
1253         list_add_tail(&reg->status_link, &pservice->waiting);
1254         list_add_tail(&reg->session_link, &session->waiting);
1255         mutex_unlock(&pservice->lock);
1256
1257         if (pservice->auto_freq)
1258                 get_reg_freq(data, reg);
1259
1260         vpu_debug_leave();
1261
1262         return reg;
1263 }
1264
1265 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1266 {
1267         struct vpu_service_info *pservice = data->pservice;
1268         struct vcodec_mem_region *mem_region = NULL, *n;
1269
1270         list_del_init(&reg->session_link);
1271         list_del_init(&reg->status_link);
1272         if (reg == pservice->reg_codec)
1273                 pservice->reg_codec = NULL;
1274         if (reg == pservice->reg_pproc)
1275                 pservice->reg_pproc = NULL;
1276
1277         /* release memory region attach to this registers table. */
1278         list_for_each_entry_safe(mem_region, n,
1279                         &reg->mem_region_list, reg_lnk) {
1280                 vcodec_iommu_unmap_iommu(data->iommu_info, reg->session,
1281                                          mem_region->hdl);
1282                 vcodec_iommu_free(data->iommu_info, reg->session,
1283                                   mem_region->hdl);
1284                 list_del_init(&mem_region->reg_lnk);
1285                 kfree(mem_region);
1286         }
1287
1288         kfree(reg);
1289 }
1290
1291 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1292                                  struct vpu_reg *reg)
1293 {
1294         vpu_debug_enter();
1295         list_del_init(&reg->status_link);
1296         list_add_tail(&reg->status_link, &pservice->running);
1297
1298         list_del_init(&reg->session_link);
1299         list_add_tail(&reg->session_link, &reg->session->running);
1300         vpu_debug_leave();
1301 }
1302
1303 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1304 {
1305         int i;
1306         u32 *dst = reg->reg;
1307
1308         vpu_debug_enter();
1309         for (i = 0; i < count; i++, src++)
1310                 *dst++ = readl_relaxed(src);
1311
1312         dst = (u32 *)&reg->reg[0];
1313         for (i = 0; i < count; i++)
1314                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1315
1316         vpu_debug_leave();
1317 }
1318
1319 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1320                                  struct vpu_reg *reg)
1321 {
1322         struct vpu_service_info *pservice = data->pservice;
1323         struct vpu_hw_info *hw_info = data->hw_info;
1324         struct vpu_task_info *task = reg->task;
1325
1326         vpu_debug_enter();
1327
1328         list_del_init(&reg->status_link);
1329         list_add_tail(&reg->status_link, &pservice->done);
1330
1331         list_del_init(&reg->session_link);
1332         list_add_tail(&reg->session_link, &reg->session->done);
1333
1334         switch (reg->type) {
1335         case VPU_ENC: {
1336                 pservice->reg_codec = NULL;
1337                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1338                 reg->reg[task->reg_irq] = pservice->irq_status;
1339         } break;
1340         case VPU_DEC: {
1341                 pservice->reg_codec = NULL;
1342                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1343
1344                 /* revert hack for decoded length */
1345                 if (task->reg_len > 0) {
1346                         int reg_len = task->reg_len;
1347                         u32 dec_get = reg->reg[reg_len];
1348                         s32 dec_length = dec_get - reg->dec_base;
1349
1350                         vpu_debug(DEBUG_REGISTER,
1351                                   "dec_get %08x dec_length %d\n",
1352                                   dec_get, dec_length);
1353                         reg->reg[reg_len] = dec_length << 10;
1354                 }
1355
1356                 reg->reg[task->reg_irq] = pservice->irq_status;
1357         } break;
1358         case VPU_PP: {
1359                 pservice->reg_pproc = NULL;
1360                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1361                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1362         } break;
1363         case VPU_DEC_PP: {
1364                 u32 pipe_mode;
1365                 u32 *regs = data->dec_dev.regs;
1366
1367                 pservice->reg_codec = NULL;
1368                 pservice->reg_pproc = NULL;
1369
1370                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1371
1372                 /* NOTE: remove pp pipeline mode flag first */
1373                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1374                 pipe_mode &= ~task->pipe_mask;
1375                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1376
1377                 /* revert hack for decoded length */
1378                 if (task->reg_len > 0) {
1379                         int reg_len = task->reg_len;
1380                         u32 dec_get = reg->reg[reg_len];
1381                         s32 dec_length = dec_get - reg->dec_base;
1382
1383                         vpu_debug(DEBUG_REGISTER,
1384                                   "dec_get %08x dec_length %d\n",
1385                                   dec_get, dec_length);
1386                         reg->reg[reg_len] = dec_length << 10;
1387                 }
1388
1389                 reg->reg[task->reg_irq] = pservice->irq_status;
1390         } break;
1391         default: {
1392                 vpu_err("error: copy reg from hw with unknown type %d\n",
1393                         reg->type);
1394         } break;
1395         }
1396         vcodec_exit_mode(data);
1397
1398         atomic_sub(1, &reg->session->task_running);
1399         atomic_sub(1, &pservice->total_running);
1400         wake_up(&reg->session->wait);
1401
1402         vpu_debug_leave();
1403 }
1404
1405 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1406                                  struct vpu_reg *reg)
1407 {
1408         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1409
1410         if (curr == reg->freq)
1411                 return;
1412
1413         atomic_set(&pservice->freq_status, reg->freq);
1414         switch (reg->freq) {
1415         case VPU_FREQ_200M: {
1416                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1417         } break;
1418         case VPU_FREQ_266M: {
1419                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1420         } break;
1421         case VPU_FREQ_300M: {
1422                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1423         } break;
1424         case VPU_FREQ_400M: {
1425                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1426         } break;
1427         case VPU_FREQ_500M: {
1428                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1429         } break;
1430         case VPU_FREQ_600M: {
1431                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1432         } break;
1433         default: {
1434                 unsigned long rate = 300*MHZ;
1435
1436                 if (of_machine_is_compatible("rockchip,rk2928g"))
1437                         rate = 400*MHZ;
1438
1439                 clk_set_rate(pservice->aclk_vcodec, rate);
1440         } break;
1441         }
1442 }
1443
1444 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1445 {
1446         struct vpu_service_info *pservice = data->pservice;
1447         struct vpu_task_info *task = reg->task;
1448         struct vpu_hw_info *hw_info = data->hw_info;
1449         int i;
1450         u32 *src = (u32 *)&reg->reg[0];
1451         u32 enable_mask = task->enable_mask;
1452         u32 gating_mask = task->gating_mask;
1453         u32 reg_en = task->reg_en;
1454
1455         vpu_debug_enter();
1456
1457         atomic_add(1, &pservice->total_running);
1458         atomic_add(1, &reg->session->task_running);
1459
1460         if (pservice->auto_freq)
1461                 vpu_service_set_freq(pservice, reg);
1462
1463         vcodec_enter_mode(data);
1464
1465         switch (reg->type) {
1466         case VPU_ENC: {
1467                 u32 *dst = data->enc_dev.regs;
1468                 u32 base = 0;
1469                 u32 end  = hw_info->enc_reg_num;
1470                 /* u32 reg_gating = task->reg_gating; */
1471
1472                 pservice->reg_codec = reg;
1473
1474                 vpu_debug(DEBUG_TASK_INFO,
1475                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1476                           base, end, reg_en, enable_mask, gating_mask);
1477
1478                 VEPU_CLEAN_CACHE(dst);
1479
1480                 if (debug & DEBUG_SET_REG)
1481                         for (i = base; i < end; i++)
1482                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1483                                           i, src[i]);
1484
1485                 /*
1486                  * NOTE: encoder need to setup mode first
1487                  */
1488                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1489
1490                 /* NOTE: encoder gating is not on enable register */
1491                 /* src[reg_gating] |= gating_mask; */
1492
1493                 for (i = base; i < end; i++) {
1494                         if (i != reg_en)
1495                                 writel_relaxed(src[i], dst + i);
1496                 }
1497
1498                 writel(src[reg_en], dst + reg_en);
1499                 dsb(sy);
1500
1501                 time_record(reg->task, 0);
1502         } break;
1503         case VPU_DEC: {
1504                 u32 *dst = data->dec_dev.regs;
1505                 u32 len = hw_info->dec_reg_num;
1506                 u32 base = hw_info->base_dec;
1507                 u32 end  = hw_info->end_dec;
1508
1509                 pservice->reg_codec = reg;
1510
1511                 vpu_debug(DEBUG_TASK_INFO,
1512                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1513                           base, end, reg_en, enable_mask, gating_mask);
1514
1515                 VDPU_CLEAN_CACHE(dst);
1516
1517                 /* on rkvdec set cache size to 64byte */
1518                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1519                         u32 *cache_base = dst + 0x100;
1520                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1521                         writel_relaxed(val, cache_base + 0x07);
1522                         writel_relaxed(val, cache_base + 0x17);
1523                 }
1524
1525                 if (debug & DEBUG_SET_REG)
1526                         for (i = 0; i < len; i++)
1527                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1528                                           i, src[i]);
1529                 /*
1530                  * NOTE: The end register is invalid. Do NOT write to it
1531                  *       Also the base register must be written
1532                  */
1533                 for (i = base; i < end; i++) {
1534                         if (i != reg_en)
1535                                 writel_relaxed(src[i], dst + i);
1536                 }
1537
1538                 writel(src[reg_en] | gating_mask, dst + reg_en);
1539                 dsb(sy);
1540
1541                 time_record(reg->task, 0);
1542         } break;
1543         case VPU_PP: {
1544                 u32 *dst = data->dec_dev.regs;
1545                 u32 base = hw_info->base_pp;
1546                 u32 end  = hw_info->end_pp;
1547
1548                 pservice->reg_pproc = reg;
1549
1550                 vpu_debug(DEBUG_TASK_INFO,
1551                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1552                           base, end, reg_en, enable_mask, gating_mask);
1553
1554                 if (debug & DEBUG_SET_REG)
1555                         for (i = base; i < end; i++)
1556                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1557                                           i, src[i]);
1558
1559                 for (i = base; i < end; i++) {
1560                         if (i != reg_en)
1561                                 writel_relaxed(src[i], dst + i);
1562                 }
1563
1564                 writel(src[reg_en] | gating_mask, dst + reg_en);
1565                 dsb(sy);
1566
1567                 time_record(reg->task, 0);
1568         } break;
1569         case VPU_DEC_PP: {
1570                 u32 *dst = data->dec_dev.regs;
1571                 u32 base = hw_info->base_dec_pp;
1572                 u32 end  = hw_info->end_dec_pp;
1573
1574                 pservice->reg_codec = reg;
1575                 pservice->reg_pproc = reg;
1576
1577                 vpu_debug(DEBUG_TASK_INFO,
1578                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1579                           base, end, reg_en, enable_mask, gating_mask);
1580
1581                 /* VDPU_SOFT_RESET(dst); */
1582                 VDPU_CLEAN_CACHE(dst);
1583
1584                 if (debug & DEBUG_SET_REG)
1585                         for (i = base; i < end; i++)
1586                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1587                                           i, src[i]);
1588
1589                 for (i = base; i < end; i++) {
1590                         if (i != reg_en)
1591                                 writel_relaxed(src[i], dst + i);
1592                 }
1593
1594                 /* NOTE: dec output must be disabled */
1595
1596                 writel(src[reg_en] | gating_mask, dst + reg_en);
1597                 dsb(sy);
1598
1599                 time_record(reg->task, 0);
1600         } break;
1601         default: {
1602                 vpu_err("error: unsupport session type %d", reg->type);
1603                 atomic_sub(1, &pservice->total_running);
1604                 atomic_sub(1, &reg->session->task_running);
1605         } break;
1606         }
1607
1608         vpu_debug_leave();
1609 }
1610
1611 static void try_set_reg(struct vpu_subdev_data *data)
1612 {
1613         struct vpu_service_info *pservice = data->pservice;
1614
1615         vpu_debug_enter();
1616
1617         mutex_lock(&pservice->shutdown_lock);
1618         if (atomic_read(&pservice->service_on) == 0) {
1619                 mutex_unlock(&pservice->shutdown_lock);
1620                 return;
1621         }
1622         if (!list_empty(&pservice->waiting)) {
1623                 struct vpu_reg *reg_codec = pservice->reg_codec;
1624                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1625                 int can_set = 0;
1626                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1627                 int reset_request = atomic_read(&pservice->reset_request);
1628                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1629                                 struct vpu_reg, status_link);
1630
1631                 if (change_able || !reset_request) {
1632                         switch (reg->type) {
1633                         case VPU_ENC: {
1634                                 if (change_able)
1635                                         can_set = 1;
1636                         } break;
1637                         case VPU_DEC: {
1638                                 if (reg_codec == NULL)
1639                                         can_set = 1;
1640                                 if (pservice->auto_freq && (reg_pproc != NULL))
1641                                         can_set = 0;
1642                         } break;
1643                         case VPU_PP: {
1644                                 if (reg_codec == NULL) {
1645                                         if (reg_pproc == NULL)
1646                                                 can_set = 1;
1647                                 } else {
1648                                         if ((reg_codec->type == VPU_DEC) &&
1649                                             (reg_pproc == NULL))
1650                                                 can_set = 1;
1651
1652                                         /*
1653                                          * NOTE:
1654                                          * can not charge frequency
1655                                          * when vpu is working
1656                                          */
1657                                         if (pservice->auto_freq)
1658                                                 can_set = 0;
1659                                 }
1660                         } break;
1661                         case VPU_DEC_PP: {
1662                                 if (change_able)
1663                                         can_set = 1;
1664                                 } break;
1665                         default: {
1666                                 dev_err(pservice->dev,
1667                                         "undefined reg type %d\n",
1668                                         reg->type);
1669                         } break;
1670                         }
1671                 }
1672
1673                 /* then check reset request */
1674                 if (reset_request && !change_able)
1675                         reset_request = 0;
1676
1677                 /* do reset before setting registers */
1678                 if (reset_request)
1679                         vpu_reset(data);
1680
1681                 if (can_set) {
1682                         reg_from_wait_to_run(pservice, reg);
1683                         reg_copy_to_hw(reg->data, reg);
1684                 }
1685         }
1686
1687         mutex_unlock(&pservice->shutdown_lock);
1688         vpu_debug_leave();
1689 }
1690
1691 static int return_reg(struct vpu_subdev_data *data,
1692                       struct vpu_reg *reg, u32 __user *dst)
1693 {
1694         struct vpu_hw_info *hw_info = data->hw_info;
1695         size_t size = reg->size;
1696         u32 base;
1697
1698         vpu_debug_enter();
1699         switch (reg->type) {
1700         case VPU_ENC: {
1701                 base = 0;
1702         } break;
1703         case VPU_DEC: {
1704                 base = hw_info->base_dec_pp;
1705         } break;
1706         case VPU_PP: {
1707                 base = hw_info->base_pp;
1708         } break;
1709         case VPU_DEC_PP: {
1710                 base = hw_info->base_dec_pp;
1711         } break;
1712         default: {
1713                 vpu_err("error: copy reg to user with unknown type %d\n",
1714                         reg->type);
1715                 return -EFAULT;
1716         } break;
1717         }
1718
1719         if (copy_to_user(dst, &reg->reg[base], size)) {
1720                 vpu_err("error: copy_to_user failed\n");
1721                 return -EFAULT;
1722         }
1723
1724         reg_deinit(data, reg);
1725         vpu_debug_leave();
1726         return 0;
1727 }
1728
1729 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1730                               unsigned long arg)
1731 {
1732         struct vpu_subdev_data *data =
1733                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1734                              struct vpu_subdev_data, cdev);
1735         struct vpu_service_info *pservice = data->pservice;
1736         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1737
1738         vpu_debug_enter();
1739         if (NULL == session)
1740                 return -EINVAL;
1741
1742         switch (cmd) {
1743         case VPU_IOC_SET_CLIENT_TYPE: {
1744                 session->type = (enum VPU_CLIENT_TYPE)arg;
1745                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1746                           session->pid, session->type);
1747         } break;
1748         case VPU_IOC_GET_HW_FUSE_STATUS: {
1749                 struct vpu_request req;
1750
1751                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1752                           session->pid, session->type);
1753                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1754                         vpu_err("error: get hw status copy_from_user failed\n");
1755                         return -EFAULT;
1756                 } else {
1757                         void *config = (session->type != VPU_ENC) ?
1758                                        ((void *)&pservice->dec_config) :
1759                                        ((void *)&pservice->enc_config);
1760                         size_t size = (session->type != VPU_ENC) ?
1761                                       (sizeof(struct vpu_dec_config)) :
1762                                       (sizeof(struct vpu_enc_config));
1763                         if (copy_to_user((void __user *)req.req,
1764                                          config, size)) {
1765                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1766                                         session->type);
1767                                 return -EFAULT;
1768                         }
1769                 }
1770         } break;
1771         case VPU_IOC_SET_REG: {
1772                 struct vpu_request req;
1773                 struct vpu_reg *reg;
1774
1775                 vpu_service_power_on(data, pservice);
1776
1777                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1778                           session->pid, session->type);
1779                 if (copy_from_user(&req, (void __user *)arg,
1780                                    sizeof(struct vpu_request))) {
1781                         vpu_err("error: set reg copy_from_user failed\n");
1782                         return -EFAULT;
1783                 }
1784
1785                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1786                 if (NULL == reg) {
1787                         return -EFAULT;
1788                 } else {
1789                         mutex_lock(&pservice->lock);
1790                         try_set_reg(data);
1791                         mutex_unlock(&pservice->lock);
1792                 }
1793         } break;
1794         case VPU_IOC_GET_REG: {
1795                 struct vpu_request req;
1796                 struct vpu_reg *reg;
1797                 int ret;
1798
1799                 vpu_service_power_on(data, pservice);
1800
1801                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1802                           session->pid, session->type);
1803                 if (copy_from_user(&req, (void __user *)arg,
1804                                    sizeof(struct vpu_request))) {
1805                         vpu_err("error: get reg copy_from_user failed\n");
1806                         return -EFAULT;
1807                 }
1808
1809                 ret = wait_event_timeout(session->wait,
1810                                          !list_empty(&session->done),
1811                                          VPU_TIMEOUT_DELAY);
1812
1813                 if (!list_empty(&session->done)) {
1814                         if (ret < 0)
1815                                 vpu_err("warning: pid %d wait task error ret %d\n",
1816                                         session->pid, ret);
1817                         ret = 0;
1818                 } else {
1819                         if (unlikely(ret < 0)) {
1820                                 vpu_err("error: pid %d wait task ret %d\n",
1821                                         session->pid, ret);
1822                         } else if (ret == 0) {
1823                                 vpu_err("error: pid %d wait %d task done timeout\n",
1824                                         session->pid,
1825                                         atomic_read(&session->task_running));
1826                                 ret = -ETIMEDOUT;
1827                         }
1828                 }
1829
1830                 if (ret < 0) {
1831                         int task_running = atomic_read(&session->task_running);
1832
1833                         mutex_lock(&pservice->lock);
1834                         vpu_service_dump(pservice);
1835                         if (task_running) {
1836                                 atomic_set(&session->task_running, 0);
1837                                 atomic_sub(task_running,
1838                                            &pservice->total_running);
1839                                 dev_err(pservice->dev,
1840                                         "%d task is running but not return, reset hardware...",
1841                                         task_running);
1842                                 vpu_reset(data);
1843                                 dev_err(pservice->dev, "done\n");
1844                         }
1845                         vpu_service_session_clear(data, session);
1846                         mutex_unlock(&pservice->lock);
1847                         return ret;
1848                 }
1849
1850                 mutex_lock(&pservice->lock);
1851                 reg = list_entry(session->done.next,
1852                                  struct vpu_reg, session_link);
1853                 return_reg(data, reg, (u32 __user *)req.req);
1854                 mutex_unlock(&pservice->lock);
1855         } break;
1856         case VPU_IOC_PROBE_IOMMU_STATUS: {
1857                 int iommu_enable = 1;
1858
1859                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1860
1861                 if (copy_to_user((void __user *)arg,
1862                                  &iommu_enable, sizeof(int))) {
1863                         vpu_err("error: iommu status copy_to_user failed\n");
1864                         return -EFAULT;
1865                 }
1866         } break;
1867         default: {
1868                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1869         } break;
1870         }
1871         vpu_debug_leave();
1872         return 0;
1873 }
1874
1875 #ifdef CONFIG_COMPAT
1876 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1877                                      unsigned long arg)
1878 {
1879         struct vpu_subdev_data *data =
1880                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1881                              struct vpu_subdev_data, cdev);
1882         struct vpu_service_info *pservice = data->pservice;
1883         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1884
1885         vpu_debug_enter();
1886         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1887                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1888         if (NULL == session)
1889                 return -EINVAL;
1890
1891         switch (cmd) {
1892         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1893                 session->type = (enum VPU_CLIENT_TYPE)arg;
1894                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1895                           session->type);
1896         } break;
1897         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1898                 struct compat_vpu_request req;
1899
1900                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1901                           session->type);
1902                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1903                                    sizeof(struct compat_vpu_request))) {
1904                         vpu_err("error: compat get hw status copy_from_user failed\n");
1905                         return -EFAULT;
1906                 } else {
1907                         void *config = (session->type != VPU_ENC) ?
1908                                        ((void *)&pservice->dec_config) :
1909                                        ((void *)&pservice->enc_config);
1910                         size_t size = (session->type != VPU_ENC) ?
1911                                       (sizeof(struct vpu_dec_config)) :
1912                                       (sizeof(struct vpu_enc_config));
1913
1914                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1915                                          config, size)) {
1916                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1917                                         session->type);
1918                                 return -EFAULT;
1919                         }
1920                 }
1921         } break;
1922         case COMPAT_VPU_IOC_SET_REG: {
1923                 struct compat_vpu_request req;
1924                 struct vpu_reg *reg;
1925
1926                 vpu_service_power_on(data, pservice);
1927
1928                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1929                           session->type);
1930                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1931                                    sizeof(struct compat_vpu_request))) {
1932                         vpu_err("compat set_reg copy_from_user failed\n");
1933                         return -EFAULT;
1934                 }
1935                 reg = reg_init(data, session,
1936                                compat_ptr((compat_uptr_t)req.req), req.size);
1937                 if (NULL == reg) {
1938                         return -EFAULT;
1939                 } else {
1940                         mutex_lock(&pservice->lock);
1941                         try_set_reg(data);
1942                         mutex_unlock(&pservice->lock);
1943                 }
1944         } break;
1945         case COMPAT_VPU_IOC_GET_REG: {
1946                 struct compat_vpu_request req;
1947                 struct vpu_reg *reg;
1948                 int ret;
1949
1950                 vpu_service_power_on(data, pservice);
1951
1952                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1953                           session->type);
1954                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1955                                    sizeof(struct compat_vpu_request))) {
1956                         vpu_err("compat get reg copy_from_user failed\n");
1957                         return -EFAULT;
1958                 }
1959
1960                 ret = wait_event_timeout(session->wait,
1961                                          !list_empty(&session->done),
1962                                          VPU_TIMEOUT_DELAY);
1963
1964                 if (!list_empty(&session->done)) {
1965                         if (ret < 0)
1966                                 vpu_err("warning: pid %d wait task error ret %d\n",
1967                                         session->pid, ret);
1968                         ret = 0;
1969                 } else {
1970                         if (unlikely(ret < 0)) {
1971                                 vpu_err("error: pid %d wait task ret %d\n",
1972                                         session->pid, ret);
1973                         } else if (ret == 0) {
1974                                 vpu_err("error: pid %d wait %d task done timeout\n",
1975                                         session->pid,
1976                                         atomic_read(&session->task_running));
1977                                 ret = -ETIMEDOUT;
1978                         }
1979                 }
1980
1981                 if (ret < 0) {
1982                         int task_running = atomic_read(&session->task_running);
1983
1984                         mutex_lock(&pservice->lock);
1985                         vpu_service_dump(pservice);
1986                         if (task_running) {
1987                                 atomic_set(&session->task_running, 0);
1988                                 atomic_sub(task_running,
1989                                            &pservice->total_running);
1990                                 dev_err(pservice->dev,
1991                                         "%d task is running but not return, reset hardware...",
1992                                         task_running);
1993                                 vpu_reset(data);
1994                                 dev_err(pservice->dev, "done\n");
1995                         }
1996                         vpu_service_session_clear(data, session);
1997                         mutex_unlock(&pservice->lock);
1998                         return ret;
1999                 }
2000
2001                 mutex_lock(&pservice->lock);
2002                 reg = list_entry(session->done.next,
2003                                  struct vpu_reg, session_link);
2004                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
2005                 mutex_unlock(&pservice->lock);
2006         } break;
2007         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
2008                 int iommu_enable = 1;
2009
2010                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
2011
2012                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
2013                                  &iommu_enable, sizeof(int))) {
2014                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2015                         return -EFAULT;
2016                 }
2017         } break;
2018         default: {
2019                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2020         } break;
2021         }
2022         vpu_debug_leave();
2023         return 0;
2024 }
2025 #endif
2026
2027 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2028 {
2029         struct vpu_service_info *pservice = data->pservice;
2030         int ret = -EINVAL, i = 0;
2031         u32 hw_id = readl_relaxed(data->regs);
2032
2033         hw_id = (hw_id >> 16) & 0xFFFF;
2034         dev_info(pservice->dev, "checking hw id %x\n", hw_id);
2035         data->hw_info = NULL;
2036
2037         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2038                 const struct vcodec_info *info = &vcodec_info_set[i];
2039
2040                 if (hw_id == info->hw_id) {
2041                         data->hw_id = info->hw_id;
2042                         data->hw_info = info->hw_info;
2043                         data->task_info = info->task_info;
2044                         data->trans_info = info->trans_info;
2045                         ret = 0;
2046                         break;
2047                 }
2048         }
2049         return ret;
2050 }
2051
2052 static int vpu_service_open(struct inode *inode, struct file *filp)
2053 {
2054         struct vpu_subdev_data *data = container_of(
2055                         inode->i_cdev, struct vpu_subdev_data, cdev);
2056         struct vpu_service_info *pservice = data->pservice;
2057         struct vpu_session *session = NULL;
2058
2059         vpu_debug_enter();
2060
2061         session = kzalloc(sizeof(*session), GFP_KERNEL);
2062         if (!session) {
2063                 vpu_err("error: unable to allocate memory for vpu_session.");
2064                 return -ENOMEM;
2065         }
2066
2067         data->iommu_info->debug_level = debug;
2068
2069         session->type   = VPU_TYPE_BUTT;
2070         session->pid    = current->pid;
2071         INIT_LIST_HEAD(&session->waiting);
2072         INIT_LIST_HEAD(&session->running);
2073         INIT_LIST_HEAD(&session->done);
2074         INIT_LIST_HEAD(&session->list_session);
2075         init_waitqueue_head(&session->wait);
2076         atomic_set(&session->task_running, 0);
2077         mutex_lock(&pservice->lock);
2078         list_add_tail(&session->list_session, &pservice->session);
2079         filp->private_data = (void *)session;
2080         mutex_unlock(&pservice->lock);
2081
2082         dev_dbg(pservice->dev, "dev opened\n");
2083         vpu_debug_leave();
2084         return nonseekable_open(inode, filp);
2085 }
2086
2087 static int vpu_service_release(struct inode *inode, struct file *filp)
2088 {
2089         struct vpu_subdev_data *data = container_of(
2090                         inode->i_cdev, struct vpu_subdev_data, cdev);
2091         struct vpu_service_info *pservice = data->pservice;
2092         int task_running;
2093         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2094
2095         vpu_debug_enter();
2096         if (NULL == session)
2097                 return -EINVAL;
2098
2099         task_running = atomic_read(&session->task_running);
2100         if (task_running) {
2101                 dev_err(pservice->dev,
2102                         "error: session %d still has %d task running when closing\n",
2103                         session->pid, task_running);
2104                 msleep(50);
2105         }
2106         wake_up(&session->wait);
2107
2108         vpu_service_power_on(data, pservice);
2109         mutex_lock(&pservice->lock);
2110         /* remove this filp from the asynchronusly notified filp's */
2111         list_del_init(&session->list_session);
2112         vpu_service_session_clear(data, session);
2113         vcodec_iommu_clear(data->iommu_info, session);
2114         kfree(session);
2115         filp->private_data = NULL;
2116         mutex_unlock(&pservice->lock);
2117
2118         dev_info(pservice->dev, "closed\n");
2119         vpu_debug_leave();
2120         return 0;
2121 }
2122
2123 static const struct file_operations vpu_service_fops = {
2124         .unlocked_ioctl = vpu_service_ioctl,
2125         .open           = vpu_service_open,
2126         .release        = vpu_service_release,
2127 #ifdef CONFIG_COMPAT
2128         .compat_ioctl   = compat_vpu_service_ioctl,
2129 #endif
2130 };
2131
2132 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2133 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2134 static irqreturn_t vepu_irq(int irq, void *dev_id);
2135 static irqreturn_t vepu_isr(int irq, void *dev_id);
2136 static void get_hw_info(struct vpu_subdev_data *data);
2137
2138 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2139 {
2140         struct device_node *dn = NULL;
2141         struct platform_device *pd = NULL;
2142         struct device *ret = NULL;
2143
2144         dn = of_find_compatible_node(NULL, NULL, compt);
2145         if (!dn) {
2146                 pr_err("can't find device node %s \r\n", compt);
2147                 return NULL;
2148         }
2149
2150         pd = of_find_device_by_node(dn);
2151         if (!pd) {
2152                 pr_err("can't find platform device in device node %s\n", compt);
2153                 return  NULL;
2154         }
2155         ret = &pd->dev;
2156
2157         return ret;
2158 }
2159
2160 #ifdef CONFIG_IOMMU_API
2161 static inline void platform_set_sysmmu(struct device *iommu,
2162                                        struct device *dev)
2163 {
2164         dev->archdata.iommu = iommu;
2165 }
2166 #else
2167 static inline void platform_set_sysmmu(struct device *iommu,
2168                                        struct device *dev)
2169 {
2170 }
2171 #endif
2172
2173 int vcodec_sysmmu_fault_hdl(struct device *dev,
2174                             enum rk_iommu_inttype itype,
2175                             unsigned long pgtable_base,
2176                             unsigned long fault_addr, unsigned int status)
2177 {
2178         struct platform_device *pdev;
2179         struct vpu_service_info *pservice;
2180         struct vpu_subdev_data *data;
2181
2182         vpu_debug_enter();
2183
2184         if (dev == NULL) {
2185                 pr_err("invalid NULL dev\n");
2186                 return 0;
2187         }
2188
2189         pdev = container_of(dev, struct platform_device, dev);
2190         if (pdev == NULL) {
2191                 pr_err("invalid NULL platform_device\n");
2192                 return 0;
2193         }
2194
2195         data = platform_get_drvdata(pdev);
2196         if (data == NULL) {
2197                 pr_err("invalid NULL vpu_subdev_data\n");
2198                 return 0;
2199         }
2200
2201         pservice = data->pservice;
2202         if (pservice == NULL) {
2203                 pr_err("invalid NULL vpu_service_info\n");
2204                 return 0;
2205         }
2206
2207         if (pservice->reg_codec) {
2208                 struct vpu_reg *reg = pservice->reg_codec;
2209                 struct vcodec_mem_region *mem, *n;
2210                 int i = 0;
2211
2212                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2213                 if (!list_empty(&reg->mem_region_list)) {
2214                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2215                                                  reg_lnk) {
2216                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2217                                        mem->reg_idx, i, mem->iova, mem->len);
2218                                 i++;
2219                         }
2220                 } else {
2221                         pr_err("no memory region mapped\n");
2222                 }
2223
2224                 if (reg->data) {
2225                         struct vpu_subdev_data *data = reg->data;
2226                         u32 *base = (u32 *)data->dec_dev.regs;
2227                         u32 len = data->hw_info->dec_reg_num;
2228
2229                         pr_err("current errror register set:\n");
2230
2231                         for (i = 0; i < len; i++)
2232                                 pr_err("reg[%02d] %08x\n",
2233                                        i, readl_relaxed(base + i));
2234                 }
2235
2236                 pr_alert("vcodec, page fault occur, reset hw\n");
2237
2238                 /* reg->reg[101] = 1; */
2239                 _vpu_reset(data);
2240         }
2241
2242         return 0;
2243 }
2244
2245 static int vcodec_subdev_probe(struct platform_device *pdev,
2246                                struct vpu_service_info *pservice)
2247 {
2248         uint8_t *regs = NULL;
2249         int32_t ret = 0;
2250         uint32_t ioaddr = 0;
2251         struct resource *res = NULL;
2252         struct vpu_hw_info *hw_info = NULL;
2253         struct device *dev = &pdev->dev;
2254         struct device_node *np = pdev->dev.of_node;
2255         struct vpu_subdev_data *data = NULL;
2256         struct platform_device *sub_dev = NULL;
2257         struct device_node *sub_np = NULL;
2258         const char *name  = np->name;
2259         char mmu_dev_dts_name[40];
2260
2261         dev_info(dev, "probe device");
2262
2263         data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2264         if (!data)
2265                 return -ENOMEM;
2266
2267         data->pservice = pservice;
2268         data->dev = dev;
2269         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2270
2271         if (pservice->reg_base == 0) {
2272                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2273                 data->regs = devm_ioremap_resource(dev, res);
2274                 if (IS_ERR(data->regs)) {
2275                         ret = PTR_ERR(data->regs);
2276                         goto err;
2277                 }
2278                 ioaddr = res->start;
2279         } else {
2280                 data->regs = pservice->reg_base;
2281                 ioaddr = pservice->ioaddr;
2282         }
2283
2284         sub_np = of_parse_phandle(np, "iommus", 0);
2285         if (sub_np) {
2286                 sub_dev = of_find_device_by_node(sub_np);
2287                 data->mmu_dev = &sub_dev->dev;
2288         }
2289
2290         /* Back to legacy iommu probe */
2291         if (!data->mmu_dev) {
2292                 switch (data->mode) {
2293                 case VCODEC_RUNNING_MODE_VPU:
2294                         sprintf(mmu_dev_dts_name,
2295                                 VPU_IOMMU_COMPATIBLE_NAME);
2296                         break;
2297                 case VCODEC_RUNNING_MODE_RKVDEC:
2298                         sprintf(mmu_dev_dts_name,
2299                                 VDEC_IOMMU_COMPATIBLE_NAME);
2300                         break;
2301                 case VCODEC_RUNNING_MODE_HEVC:
2302                 default:
2303                         sprintf(mmu_dev_dts_name,
2304                                 HEVC_IOMMU_COMPATIBLE_NAME);
2305                         break;
2306                 }
2307
2308                 data->mmu_dev =
2309                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2310                 if (data->mmu_dev)
2311                         platform_set_sysmmu(data->mmu_dev, dev);
2312
2313                 rockchip_iovmm_set_fault_handler
2314                         (dev, vcodec_sysmmu_fault_hdl);
2315         }
2316
2317         dev_info(dev, "vpu mmu dec %p\n", data->mmu_dev);
2318
2319         clear_bit(MMU_ACTIVATED, &data->state);
2320         vpu_service_power_on(data, pservice);
2321
2322         vcodec_enter_mode(data);
2323         ret = vpu_service_check_hw(data);
2324         if (ret < 0) {
2325                 vpu_err("error: hw info check faild\n");
2326                 goto err;
2327         }
2328
2329         hw_info = data->hw_info;
2330         regs = (u8 *)data->regs;
2331
2332         if (hw_info->dec_reg_num) {
2333                 data->dec_dev.iosize = hw_info->dec_io_size;
2334                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2335         }
2336
2337         if (hw_info->enc_reg_num) {
2338                 data->enc_dev.iosize = hw_info->enc_io_size;
2339                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2340         }
2341
2342         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2343
2344         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2345         if (data->irq_enc > 0) {
2346                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2347                                                 vepu_irq, vepu_isr,
2348                                                 IRQF_SHARED, dev_name(dev),
2349                                                 (void *)data);
2350                 if (ret) {
2351                         dev_err(dev, "error: can't request vepu irq %d\n",
2352                                 data->irq_enc);
2353                         goto err;
2354                 }
2355         }
2356         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2357         if (data->irq_dec > 0) {
2358                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2359                                                 vdpu_irq, vdpu_isr,
2360                                                 IRQF_SHARED, dev_name(dev),
2361                                                 (void *)data);
2362                 if (ret) {
2363                         dev_err(dev, "error: can't request vdpu irq %d\n",
2364                                 data->irq_dec);
2365                         goto err;
2366                 }
2367         }
2368         atomic_set(&data->dec_dev.irq_count_codec, 0);
2369         atomic_set(&data->dec_dev.irq_count_pp, 0);
2370         atomic_set(&data->enc_dev.irq_count_codec, 0);
2371         atomic_set(&data->enc_dev.irq_count_pp, 0);
2372
2373         of_property_read_u32(np, "allocator", (u32 *)&pservice->alloc_type);
2374         data->iommu_info = vcodec_iommu_info_create(dev, data->mmu_dev,
2375                                                     pservice->alloc_type);
2376         dev_info(dev, "allocator is %s\n", pservice->alloc_type == 1 ? "drm" :
2377                 (pservice->alloc_type == 2 ? "ion" : "null"));
2378         get_hw_info(data);
2379         pservice->auto_freq = true;
2380
2381         vcodec_exit_mode(data);
2382         /* create device node */
2383         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2384         if (ret) {
2385                 dev_err(dev, "alloc dev_t failed\n");
2386                 goto err;
2387         }
2388
2389         cdev_init(&data->cdev, &vpu_service_fops);
2390
2391         data->cdev.owner = THIS_MODULE;
2392         data->cdev.ops = &vpu_service_fops;
2393
2394         ret = cdev_add(&data->cdev, data->dev_t, 1);
2395
2396         if (ret) {
2397                 dev_err(dev, "add dev_t failed\n");
2398                 goto err;
2399         }
2400
2401         data->cls = class_create(THIS_MODULE, name);
2402
2403         if (IS_ERR(data->cls)) {
2404                 ret = PTR_ERR(data->cls);
2405                 dev_err(dev, "class_create err:%d\n", ret);
2406                 goto err;
2407         }
2408
2409         data->child_dev = device_create(data->cls, dev,
2410                 data->dev_t, "%s", name);
2411
2412         platform_set_drvdata(pdev, data);
2413
2414         INIT_LIST_HEAD(&data->lnk_service);
2415         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2416
2417         return 0;
2418 err:
2419         if (data->child_dev) {
2420                 device_destroy(data->cls, data->dev_t);
2421                 cdev_del(&data->cdev);
2422                 unregister_chrdev_region(data->dev_t, 1);
2423         }
2424
2425         if (data->cls)
2426                 class_destroy(data->cls);
2427         return -1;
2428 }
2429
2430 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2431 {
2432         struct vpu_service_info *pservice = data->pservice;
2433
2434         vcodec_iommu_info_destroy(data->iommu_info);
2435         data->iommu_info = NULL;
2436
2437         mutex_lock(&pservice->lock);
2438         cancel_delayed_work_sync(&pservice->power_off_work);
2439         vpu_service_power_off(pservice);
2440         mutex_unlock(&pservice->lock);
2441
2442         device_destroy(data->cls, data->dev_t);
2443         class_destroy(data->cls);
2444         cdev_del(&data->cdev);
2445         unregister_chrdev_region(data->dev_t, 1);
2446
2447 #ifdef CONFIG_DEBUG_FS
2448         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2449                 debugfs_remove_recursive(data->debugfs_dir);
2450 #endif
2451 }
2452
2453 static void vcodec_read_property(struct device_node *np,
2454                                  struct vpu_service_info *pservice)
2455 {
2456         pservice->mode_bit = 0;
2457         pservice->mode_ctrl = 0;
2458         pservice->subcnt = 0;
2459         pservice->grf_base = NULL;
2460
2461         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2462
2463         if (pservice->subcnt > 1) {
2464                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2465                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2466         }
2467 #ifdef CONFIG_MFD_SYSCON
2468         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2469         if (IS_ERR_OR_NULL(pservice->grf)) {
2470                 pservice->grf = NULL;
2471 #ifdef CONFIG_ARM
2472                 pservice->grf_base = RK_GRF_VIRT;
2473 #else
2474                 vpu_err("can't find vpu grf property\n");
2475                 return;
2476 #endif
2477         }
2478 #else
2479 #ifdef CONFIG_ARM
2480         pservice->grf_base = RK_GRF_VIRT;
2481 #else
2482         vpu_err("can't find vpu grf property\n");
2483         return;
2484 #endif
2485 #endif
2486
2487 #ifdef CONFIG_RESET_CONTROLLER
2488         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2489         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2490         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2491
2492         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2493                 dev_warn(pservice->dev, "No aclk reset resource define\n");
2494                 pservice->rst_a = NULL;
2495         }
2496
2497         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2498                 dev_warn(pservice->dev, "No hclk reset resource define\n");
2499                 pservice->rst_h = NULL;
2500         }
2501
2502         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2503                 dev_warn(pservice->dev, "No core reset resource define\n");
2504                 pservice->rst_v = NULL;
2505         }
2506 #endif
2507
2508         of_property_read_string(np, "name", (const char **)&pservice->name);
2509 }
2510
2511 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2512 {
2513         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2514         pservice->curr_mode = -1;
2515
2516         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2517         INIT_LIST_HEAD(&pservice->waiting);
2518         INIT_LIST_HEAD(&pservice->running);
2519         mutex_init(&pservice->lock);
2520         mutex_init(&pservice->shutdown_lock);
2521         atomic_set(&pservice->service_on, 1);
2522
2523         INIT_LIST_HEAD(&pservice->done);
2524         INIT_LIST_HEAD(&pservice->session);
2525         INIT_LIST_HEAD(&pservice->subdev_list);
2526
2527         pservice->reg_pproc     = NULL;
2528         atomic_set(&pservice->total_running, 0);
2529         atomic_set(&pservice->enabled,       0);
2530         atomic_set(&pservice->power_on_cnt,  0);
2531         atomic_set(&pservice->power_off_cnt, 0);
2532         atomic_set(&pservice->reset_request, 0);
2533
2534         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2535         pservice->last.tv64 = 0;
2536
2537         pservice->alloc_type = 0;
2538 }
2539
2540 static int vcodec_probe(struct platform_device *pdev)
2541 {
2542         int i;
2543         int ret = 0;
2544         struct resource *res = NULL;
2545         struct device *dev = &pdev->dev;
2546         struct device_node *np = pdev->dev.of_node;
2547         struct vpu_service_info *pservice = NULL;
2548         struct vcodec_device_info *driver_data;
2549
2550         pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info),
2551                                 GFP_KERNEL);
2552         if (!pservice)
2553                 return -ENOMEM;
2554         pservice->dev = dev;
2555
2556         driver_data = vcodec_get_drv_data(pdev);
2557         if (!driver_data)
2558                 return -EINVAL;
2559
2560         vcodec_read_property(np, pservice);
2561         vcodec_init_drvdata(pservice);
2562
2563         /* Underscore for label, hyphens for name */
2564         switch (driver_data->device_type) {
2565         case VCODEC_DEVICE_TYPE_VPUX:
2566                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2567                 break;
2568         case VCODEC_DEVICE_TYPE_VPUC:
2569                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2570                 break;
2571         case VCODEC_DEVICE_TYPE_HEVC:
2572                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2573                 break;
2574         case VCODEC_DEVICE_TYPE_RKVD:
2575                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2576                 break;
2577         default:
2578                 dev_err(dev, "unsupported device type\n");
2579                 return -ENODEV;
2580         }
2581
2582         if (0 > vpu_get_clk(pservice))
2583                 goto err;
2584
2585         if (of_property_read_bool(np, "reg")) {
2586                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2587
2588                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2589                 if (IS_ERR(pservice->reg_base)) {
2590                         vpu_err("ioremap registers base failed\n");
2591                         ret = PTR_ERR(pservice->reg_base);
2592                         goto err;
2593                 }
2594                 pservice->ioaddr = res->start;
2595         } else {
2596                 pservice->reg_base = 0;
2597         }
2598
2599         pm_runtime_enable(dev);
2600
2601         if (of_property_read_bool(np, "subcnt")) {
2602                 struct vpu_subdev_data *data = NULL;
2603
2604                 data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data),
2605                                     GFP_KERNEL);
2606                 if (!data)
2607                         return -ENOMEM;
2608
2609                 for (i = 0; i < pservice->subcnt; i++) {
2610                         struct device_node *sub_np;
2611                         struct platform_device *sub_pdev;
2612
2613                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2614                         sub_pdev = of_find_device_by_node(sub_np);
2615
2616                         vcodec_subdev_probe(sub_pdev, pservice);
2617                 }
2618                 data->pservice = pservice;
2619                 platform_set_drvdata(pdev, data);
2620         } else {
2621                 vcodec_subdev_probe(pdev, pservice);
2622         }
2623
2624         vpu_service_power_off(pservice);
2625
2626         dev_info(dev, "init success\n");
2627
2628         return 0;
2629
2630 err:
2631         dev_info(dev, "init failed\n");
2632         vpu_service_power_off(pservice);
2633         wake_lock_destroy(&pservice->wake_lock);
2634
2635         return ret;
2636 }
2637
2638 static int vcodec_remove(struct platform_device *pdev)
2639 {
2640         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2641
2642         vcodec_subdev_remove(data);
2643
2644         pm_runtime_disable(data->pservice->dev);
2645
2646         return 0;
2647 }
2648
2649 static void vcodec_shutdown(struct platform_device *pdev)
2650 {
2651         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2652         struct vpu_service_info *pservice = data->pservice;
2653         struct device_node *np = pdev->dev.of_node;
2654         int val;
2655         int ret;
2656         int i;
2657
2658         dev_info(&pdev->dev, "vcodec shutdown");
2659
2660         mutex_lock(&pservice->shutdown_lock);
2661         atomic_set(&pservice->service_on, 0);
2662         mutex_unlock(&pservice->shutdown_lock);
2663
2664         ret = readx_poll_timeout(atomic_read,
2665                                  &pservice->total_running,
2666                                  val, val == 0, 20000, 200000);
2667         if (ret == -ETIMEDOUT)
2668                 dev_err(&pdev->dev, "wait total running time out\n");
2669
2670         vcodec_exit_mode(data);
2671
2672         vpu_service_clear(data);
2673         if (of_property_read_bool(np, "subcnt")) {
2674                 for (i = 0; i < pservice->subcnt; i++) {
2675                         struct device_node *sub_np;
2676                         struct platform_device *sub_pdev;
2677
2678                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2679                         sub_pdev = of_find_device_by_node(sub_np);
2680
2681                         vcodec_subdev_remove(platform_get_drvdata(sub_pdev));
2682                 }
2683
2684         } else {
2685                 vcodec_subdev_remove(data);
2686         }
2687
2688         pm_runtime_disable(&pdev->dev);
2689 }
2690
2691 static const struct of_device_id vcodec_service_dt_ids[] = {
2692         {
2693                 .compatible = "rockchip,vpu_service",
2694                 .data = &vpu_device_info,
2695         },
2696         {
2697                 .compatible = "rockchip,hevc_service",
2698                 .data = &hevc_device_info,
2699         },
2700         {
2701                 .compatible = "rockchip,vpu_combo",
2702                 .data = &vpu_combo_device_info,
2703         },
2704         {
2705                 .compatible = "rockchip,rkvdec",
2706                 .data = &rkvd_device_info,
2707         },
2708         {},
2709 };
2710
2711 MODULE_DEVICE_TABLE(of, vcodec_service_dt_ids);
2712
2713 static void *vcodec_get_drv_data(struct platform_device *pdev)
2714 {
2715         struct vcodec_device_info *driver_data = NULL;
2716         const struct of_device_id *match;
2717
2718         match = of_match_node(vcodec_service_dt_ids, pdev->dev.of_node);
2719         if (match)
2720                 driver_data = (struct vcodec_device_info *)match->data;
2721
2722         return driver_data;
2723 }
2724
2725 static struct platform_driver vcodec_driver = {
2726         .probe = vcodec_probe,
2727         .remove = vcodec_remove,
2728         .shutdown = vcodec_shutdown,
2729         .driver = {
2730                 .name = "rk-vcodec",
2731                 .owner = THIS_MODULE,
2732                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2733         },
2734 };
2735
2736 static void get_hw_info(struct vpu_subdev_data *data)
2737 {
2738         struct vpu_service_info *pservice = data->pservice;
2739         struct vpu_dec_config *dec = &pservice->dec_config;
2740         struct vpu_enc_config *enc = &pservice->enc_config;
2741
2742         if (of_machine_is_compatible("rockchip,rk2928") ||
2743                         of_machine_is_compatible("rockchip,rk3036") ||
2744                         of_machine_is_compatible("rockchip,rk3066") ||
2745                         of_machine_is_compatible("rockchip,rk3126") ||
2746                         of_machine_is_compatible("rockchip,rk3188"))
2747                 dec->max_dec_pic_width = 1920;
2748         else
2749                 dec->max_dec_pic_width = 4096;
2750
2751         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2752                 dec->h264_support = 3;
2753                 dec->jpeg_support = 1;
2754                 dec->mpeg4_support = 2;
2755                 dec->vc1_support = 3;
2756                 dec->mpeg2_support = 1;
2757                 dec->pp_support = 1;
2758                 dec->sorenson_support = 1;
2759                 dec->ref_buf_support = 3;
2760                 dec->vp6_support = 1;
2761                 dec->vp7_support = 1;
2762                 dec->vp8_support = 1;
2763                 dec->avs_support = 1;
2764                 dec->jpeg_ext_support = 0;
2765                 dec->custom_mpeg4_support = 1;
2766                 dec->reserve = 0;
2767                 dec->mvc_support = 1;
2768
2769                 if (!of_machine_is_compatible("rockchip,rk3036")) {
2770                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2771
2772                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2773                         enc->h264_enabled = 1;
2774                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2775                         enc->jpeg_enabled = 1;
2776                         enc->vs_enabled = (config_reg >> 24) & 1;
2777                         enc->rgb_enabled = (config_reg >> 28) & 1;
2778                         enc->reg_size = data->reg_size;
2779                         enc->reserv[0] = 0;
2780                         enc->reserv[1] = 0;
2781                 }
2782
2783                 pservice->auto_freq = true;
2784                 vpu_debug(DEBUG_EXTRA_INFO,
2785                           "vpu_service set to auto frequency mode\n");
2786                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2787
2788                 pservice->bug_dec_addr = of_machine_is_compatible
2789                         ("rockchip,rk30xx");
2790         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2791                 pservice->auto_freq = true;
2792                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2793         } else {
2794                 /* disable frequency switch in hevc.*/
2795                 pservice->auto_freq = false;
2796         }
2797 }
2798
2799 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2800 {
2801         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2802                   task->name, irq_status, task->error_mask);
2803
2804         return (task->error_mask & irq_status) ? true : false;
2805 }
2806
2807 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2808 {
2809         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2810         struct vpu_service_info *pservice = data->pservice;
2811         struct vpu_task_info *task = NULL;
2812         struct vpu_device *dev = &data->dec_dev;
2813         u32 hw_id = data->hw_info->hw_id;
2814         u32 raw_status;
2815         u32 dec_status;
2816
2817         task = &data->task_info[TASK_DEC];
2818
2819         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2820         dec_status = raw_status;
2821
2822         vpu_debug(DEBUG_TASK_INFO,
2823                   "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2824                   task->reg_irq, dec_status,
2825                   task->irq_mask, task->ready_mask, task->error_mask);
2826
2827         if (dec_status & task->irq_mask) {
2828                 time_record(task, 1);
2829                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2830                           dec_status);
2831                 if ((dec_status & 0x40001) == 0x40001) {
2832                         do {
2833                                 dec_status =
2834                                         readl_relaxed(dev->regs +
2835                                                 task->reg_irq);
2836                         } while ((dec_status & 0x40001) == 0x40001);
2837                 }
2838
2839                 if (check_irq_err(task, dec_status))
2840                         atomic_add(1, &pservice->reset_request);
2841
2842                 writel_relaxed(0, dev->regs + task->reg_irq);
2843
2844                 /* set clock gating to save power */
2845                 writel(task->gating_mask, dev->regs + task->reg_en);
2846
2847                 atomic_add(1, &dev->irq_count_codec);
2848                 time_diff(task);
2849         }
2850
2851         task = &data->task_info[TASK_PP];
2852         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2853                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2854
2855                 if (pp_status & task->irq_mask) {
2856                         time_record(task, 1);
2857                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2858                                   pp_status);
2859
2860                         if (check_irq_err(task, dec_status))
2861                                 atomic_add(1, &pservice->reset_request);
2862
2863                         /* clear pp IRQ */
2864                         writel_relaxed(pp_status & (~task->reg_irq),
2865                                        dev->regs + task->irq_mask);
2866                         atomic_add(1, &dev->irq_count_pp);
2867                         time_diff(task);
2868                 }
2869         }
2870
2871         pservice->irq_status = raw_status;
2872
2873         if (atomic_read(&dev->irq_count_pp) ||
2874             atomic_read(&dev->irq_count_codec))
2875                 return IRQ_WAKE_THREAD;
2876         else
2877                 return IRQ_NONE;
2878 }
2879
2880 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2881 {
2882         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2883         struct vpu_service_info *pservice = data->pservice;
2884         struct vpu_device *dev = &data->dec_dev;
2885
2886         mutex_lock(&pservice->lock);
2887         if (atomic_read(&dev->irq_count_codec)) {
2888                 atomic_sub(1, &dev->irq_count_codec);
2889                 if (pservice->reg_codec == NULL) {
2890                         vpu_err("error: dec isr with no task waiting\n");
2891                 } else {
2892                         reg_from_run_to_done(data, pservice->reg_codec);
2893                         /* avoid vpu timeout and can't recover problem */
2894                         if (data->mode == VCODEC_RUNNING_MODE_VPU)
2895                                 VDPU_SOFT_RESET(data->regs);
2896                 }
2897         }
2898
2899         if (atomic_read(&dev->irq_count_pp)) {
2900                 atomic_sub(1, &dev->irq_count_pp);
2901                 if (pservice->reg_pproc == NULL)
2902                         vpu_err("error: pp isr with no task waiting\n");
2903                 else
2904                         reg_from_run_to_done(data, pservice->reg_pproc);
2905         }
2906         try_set_reg(data);
2907         mutex_unlock(&pservice->lock);
2908         return IRQ_HANDLED;
2909 }
2910
2911 static irqreturn_t vepu_irq(int irq, void *dev_id)
2912 {
2913         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2914         struct vpu_service_info *pservice = data->pservice;
2915         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2916         struct vpu_device *dev = &data->enc_dev;
2917         u32 irq_status;
2918
2919         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2920
2921         vpu_debug(DEBUG_TASK_INFO,
2922                   "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2923                   task->reg_irq, irq_status,
2924                   task->irq_mask, task->ready_mask, task->error_mask);
2925
2926         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2927
2928         if (likely(irq_status & task->irq_mask)) {
2929                 time_record(task, 1);
2930
2931                 if (check_irq_err(task, irq_status))
2932                         atomic_add(1, &pservice->reset_request);
2933
2934                 /* clear enc IRQ */
2935                 writel_relaxed(irq_status & (~task->irq_mask),
2936                                dev->regs + task->reg_irq);
2937
2938                 atomic_add(1, &dev->irq_count_codec);
2939                 time_diff(task);
2940         }
2941
2942         pservice->irq_status = irq_status;
2943
2944         if (atomic_read(&dev->irq_count_codec))
2945                 return IRQ_WAKE_THREAD;
2946         else
2947                 return IRQ_NONE;
2948 }
2949
2950 static irqreturn_t vepu_isr(int irq, void *dev_id)
2951 {
2952         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2953         struct vpu_service_info *pservice = data->pservice;
2954         struct vpu_device *dev = &data->enc_dev;
2955
2956         mutex_lock(&pservice->lock);
2957         if (atomic_read(&dev->irq_count_codec)) {
2958                 atomic_sub(1, &dev->irq_count_codec);
2959                 if (NULL == pservice->reg_codec)
2960                         vpu_err("error: enc isr with no task waiting\n");
2961                 else
2962                         reg_from_run_to_done(data, pservice->reg_codec);
2963         }
2964         try_set_reg(data);
2965         mutex_unlock(&pservice->lock);
2966         return IRQ_HANDLED;
2967 }
2968
2969 module_platform_driver(vcodec_driver);
2970 MODULE_LICENSE("GPL v2");