drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming, chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *         Jung Zhao, jung.zhao@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/clk.h>
  21 #include <linux/compat.h>
  22 #include <linux/delay.h>
  23 #include <linux/init.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/mm.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/reset.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/wakelock.h>
  33 #include <linux/cdev.h>
  34 #include <linux/of.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/regmap.h>
  38 #include <linux/mfd/syscon.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/iopoll.h>
  43
  44 #include <linux/rockchip/cru.h>
  45 #include <linux/rockchip/pmu.h>
  46 #include <linux/rockchip/grf.h>
  47
  48 #include <linux/dma-buf.h>
  49 #include <linux/rockchip-iovmm.h>
  50
  51 #include "vcodec_hw_info.h"
  52 #include "vcodec_hw_vpu.h"
  53 #include "vcodec_hw_rkv.h"
  54 #include "vcodec_hw_vpu2.h"
  55
  56 #include "vcodec_service.h"
  57
  58 #include "vcodec_iommu_ops.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 #define MHZ                                     (1000 * 1000)
  90 #define SIZE_REG(reg)                           ((reg) * 4)
  91
  92 #define VCODEC_CLOCK_ENABLE     1
  93 #define EXTRA_INFO_MAGIC        0x4C4A46
  94
  95 static int debug;
  96 module_param(debug, int, S_IRUGO | S_IWUSR);
  97 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  98 /*
  99  * hardware information organization
 100  *
 101  * In order to support multiple hardware with different version the hardware
 102  * information is organized as follow:
 103  *
 104  * 1. First, index hardware by register size / position.
 105  *    These information is fix for each hardware and do not relate to runtime
 106  *    work flow. It only related to resource allocation.
 107  *    Descriptor: struct vpu_hw_info
 108  *
 109  * 2. Then, index hardware by runtime configuration
 110  *    These information is related to runtime setting behave including enable
 111  *    register, irq register and other key control flag
 112  *    Descriptor: struct vpu_task_info
 113  *
 114  * 3. Final, on iommu case the fd translation is required
 115  *    Descriptor: struct vpu_trans_info
 116  */
 117
 118 enum VPU_FREQ {
 119         VPU_FREQ_200M,
 120         VPU_FREQ_266M,
 121         VPU_FREQ_300M,
 122         VPU_FREQ_400M,
 123         VPU_FREQ_500M,
 124         VPU_FREQ_600M,
 125         VPU_FREQ_DEFAULT,
 126         VPU_FREQ_BUT,
 127 };
 128
 129 struct extra_info_elem {
 130         u32 index;
 131         u32 offset;
 132 };
 133
 134
 135 struct extra_info_for_iommu {
 136         u32 magic;
 137         u32 cnt;
 138         struct extra_info_elem elem[20];
 139 };
 140
 141 static const struct vcodec_info vcodec_info_set[] = {
 142         {
 143                 .hw_id          = VPU_ID_8270,
 144                 .hw_info        = &hw_vpu_8270,
 145                 .task_info      = task_vpu,
 146                 .trans_info     = trans_vpu,
 147         },
 148         {
 149                 .hw_id          = VPU_ID_4831,
 150                 .hw_info        = &hw_vpu_4831,
 151                 .task_info      = task_vpu,
 152                 .trans_info     = trans_vpu,
 153         },
 154         {
 155                 .hw_id          = VPU_DEC_ID_9190,
 156                 .hw_info        = &hw_vpu_9190,
 157                 .task_info      = task_vpu,
 158                 .trans_info     = trans_vpu,
 159         },
 160         {
 161                 .hw_id          = HEVC_ID,
 162                 .hw_info        = &hw_rkhevc,
 163                 .task_info      = task_rkv,
 164                 .trans_info     = trans_rkv,
 165         },
 166         {
 167                 .hw_id          = RKV_DEC_ID,
 168                 .hw_info        = &hw_rkvdec,
 169                 .task_info      = task_rkv,
 170                 .trans_info     = trans_rkv,
 171         },
 172         {
 173                 .hw_id          = VPU2_ID,
 174                 .hw_info        = &hw_vpu2,
 175                 .task_info      = task_vpu2,
 176                 .trans_info     = trans_vpu2,
 177         },
 178 };
 179
 180 /* Both VPU1 and VPU2 */
 181 static const struct vcodec_device_info vpu_device_info = {
 182         .device_type = VCODEC_DEVICE_TYPE_VPUX,
 183         .name = "vpu-service",
 184 };
 185
 186 static const struct vcodec_device_info vpu_combo_device_info = {
 187         .device_type = VCODEC_DEVICE_TYPE_VPUC,
 188         .name = "vpu-combo",
 189 };
 190
 191 static const struct vcodec_device_info hevc_device_info = {
 192         .device_type = VCODEC_DEVICE_TYPE_HEVC,
 193         .name = "hevc-service",
 194 };
 195
 196 static const struct vcodec_device_info rkvd_device_info = {
 197         .device_type = VCODEC_DEVICE_TYPE_RKVD,
 198         .name = "rkvdec",
 199 };
 200
 201 #define DEBUG
 202 #ifdef DEBUG
 203 #define vpu_debug_func(type, fmt, args...)                      \
 204         do {                                                    \
 205                 if (unlikely(debug & type)) {                   \
 206                         pr_info("%s:%d: " fmt,                  \
 207                                  __func__, __LINE__, ##args);   \
 208                 }                                               \
 209         } while (0)
 210 #define vpu_debug(type, fmt, args...)                           \
 211         do {                                                    \
 212                 if (unlikely(debug & type)) {                   \
 213                         pr_info(fmt, ##args);                   \
 214                 }                                               \
 215         } while (0)
 216 #else
 217 #define vpu_debug_func(level, fmt, args...)
 218 #define vpu_debug(level, fmt, args...)
 219 #endif
 220
 221 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 222 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 223
 224 #define vpu_err(fmt, args...)                           \
 225                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 226
 227 enum VPU_DEC_FMT {
 228         VPU_DEC_FMT_H264,
 229         VPU_DEC_FMT_MPEG4,
 230         VPU_DEC_FMT_H263,
 231         VPU_DEC_FMT_JPEG,
 232         VPU_DEC_FMT_VC1,
 233         VPU_DEC_FMT_MPEG2,
 234         VPU_DEC_FMT_MPEG1,
 235         VPU_DEC_FMT_VP6,
 236         VPU_DEC_FMT_RESERV0,
 237         VPU_DEC_FMT_VP7,
 238         VPU_DEC_FMT_VP8,
 239         VPU_DEC_FMT_AVS,
 240         VPU_DEC_FMT_RES
 241 };
 242
 243 /**
 244  * struct for process session which connect to vpu
 245  *
 246  * @author ChenHengming (2011-5-3)
 247  */
 248 struct vpu_session {
 249         enum VPU_CLIENT_TYPE type;
 250         /* a linked list of data so we can access them for debugging */
 251         struct list_head list_session;
 252         /* a linked list of register data waiting for process */
 253         struct list_head waiting;
 254         /* a linked list of register data in processing */
 255         struct list_head running;
 256         /* a linked list of register data processed */
 257         struct list_head done;
 258         wait_queue_head_t wait;
 259         pid_t pid;
 260         atomic_t task_running;
 261 };
 262
 263 /**
 264  * struct for process register set
 265  *
 266  * @author ChenHengming (2011-5-4)
 267  */
 268 struct vpu_reg {
 269         enum VPU_CLIENT_TYPE type;
 270         enum VPU_FREQ freq;
 271         struct vpu_session *session;
 272         struct vpu_subdev_data *data;
 273         struct vpu_task_info *task;
 274         const struct vpu_trans_info *trans;
 275
 276         /* link to vpu service session */
 277         struct list_head session_link;
 278         /* link to register set list */
 279         struct list_head status_link;
 280
 281         unsigned long size;
 282         struct list_head mem_region_list;
 283         u32 dec_base;
 284         u32 *reg;
 285 };
 286
 287 struct vpu_device {
 288         atomic_t irq_count_codec;
 289         atomic_t irq_count_pp;
 290         unsigned int iosize;
 291         u32 *regs;
 292 };
 293
 294 enum vcodec_device_id {
 295         VCODEC_DEVICE_ID_VPU,
 296         VCODEC_DEVICE_ID_HEVC,
 297         VCODEC_DEVICE_ID_COMBO,
 298         VCODEC_DEVICE_ID_RKVDEC,
 299         VCODEC_DEVICE_ID_BUTT
 300 };
 301
 302 enum VCODEC_RUNNING_MODE {
 303         VCODEC_RUNNING_MODE_NONE = -1,
 304         VCODEC_RUNNING_MODE_VPU,
 305         VCODEC_RUNNING_MODE_HEVC,
 306         VCODEC_RUNNING_MODE_RKVDEC
 307 };
 308
 309 struct vcodec_mem_region {
 310         struct list_head srv_lnk;
 311         struct list_head reg_lnk;
 312         struct list_head session_lnk;
 313         unsigned long iova;     /* virtual address for iommu */
 314         unsigned long len;
 315         u32 reg_idx;
 316         int hdl;
 317 };
 318
 319 enum vpu_ctx_state {
 320         MMU_ACTIVATED   = BIT(0)
 321 };
 322
 323 struct vpu_subdev_data {
 324         struct cdev cdev;
 325         dev_t dev_t;
 326         struct class *cls;
 327         struct device *child_dev;
 328
 329         int irq_enc;
 330         int irq_dec;
 331         struct vpu_service_info *pservice;
 332
 333         u32 *regs;
 334         enum VCODEC_RUNNING_MODE mode;
 335         struct list_head lnk_service;
 336
 337         struct device *dev;
 338
 339         struct vpu_device enc_dev;
 340         struct vpu_device dec_dev;
 341
 342         enum VPU_HW_ID hw_id;
 343         struct vpu_hw_info *hw_info;
 344         struct vpu_task_info *task_info;
 345         const struct vpu_trans_info *trans_info;
 346
 347         u32 reg_size;
 348         unsigned long state;
 349
 350 #ifdef CONFIG_DEBUG_FS
 351         struct dentry *debugfs_dir;
 352         struct dentry *debugfs_file_regs;
 353 #endif
 354
 355         struct device *mmu_dev;
 356         struct vcodec_iommu_info *iommu_info;
 357 };
 358
 359 struct vpu_service_info {
 360         struct wake_lock wake_lock;
 361         struct delayed_work power_off_work;
 362         ktime_t last; /* record previous power-on time */
 363         /* vpu service structure global lock */
 364         struct mutex lock;
 365         /* link to link_reg in struct vpu_reg */
 366         struct list_head waiting;
 367         /* link to link_reg in struct vpu_reg */
 368         struct list_head running;
 369         /* link to link_reg in struct vpu_reg */
 370         struct list_head done;
 371         /* link to list_session in struct vpu_session */
 372         struct list_head session;
 373         atomic_t total_running;
 374         atomic_t enabled;
 375         atomic_t power_on_cnt;
 376         atomic_t power_off_cnt;
 377         atomic_t service_on;
 378         struct mutex shutdown_lock;
 379         struct vpu_reg *reg_codec;
 380         struct vpu_reg *reg_pproc;
 381         struct vpu_reg *reg_resev;
 382         struct vpu_dec_config dec_config;
 383         struct vpu_enc_config enc_config;
 384
 385         bool auto_freq;
 386         bool bug_dec_addr;
 387         atomic_t freq_status;
 388
 389         struct clk *aclk_vcodec;
 390         struct clk *hclk_vcodec;
 391         struct clk *clk_core;
 392         struct clk *clk_cabac;
 393         struct clk *pd_video;
 394
 395 #ifdef CONFIG_RESET_CONTROLLER
 396         struct reset_control *rst_a;
 397         struct reset_control *rst_h;
 398         struct reset_control *rst_v;
 399 #endif
 400         struct device *dev;
 401
 402         u32 irq_status;
 403         atomic_t reset_request;
 404         struct list_head mem_region_list;
 405
 406         enum vcodec_device_id dev_id;
 407
 408         enum VCODEC_RUNNING_MODE curr_mode;
 409         u32 prev_mode;
 410
 411         struct delayed_work simulate_work;
 412
 413         u32 mode_bit;
 414         u32 mode_ctrl;
 415         u32 *reg_base;
 416         u32 ioaddr;
 417         struct regmap *grf;
 418         u32 *grf_base;
 419
 420         char *name;
 421
 422         u32 subcnt;
 423         struct list_head subdev_list;
 424
 425         u32 alloc_type;
 426 };
 427
 428 struct vpu_request {
 429         u32 *req;
 430         u32 size;
 431 };
 432
 433 #ifdef CONFIG_COMPAT
 434 struct compat_vpu_request {
 435         compat_uptr_t req;
 436         u32 size;
 437 };
 438 #endif
 439
 440 #define VDPU_SOFT_RESET_REG     101
 441 #define VDPU_CLEAN_CACHE_REG    516
 442 #define VEPU_CLEAN_CACHE_REG    772
 443 #define HEVC_CLEAN_CACHE_REG    260
 444
 445 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 446
 447 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 448 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 449 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 450 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 451
 452 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 453 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 454
 455 static void *vcodec_get_drv_data(struct platform_device *pdev);
 456
 457 static void vpu_service_power_on(struct vpu_subdev_data *data,
 458                                  struct vpu_service_info *pservice);
 459
 460 static void time_record(struct vpu_task_info *task, int is_end)
 461 {
 462         if (unlikely(debug & DEBUG_TIMING) && task)
 463                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 464 }
 465
 466 static void time_diff(struct vpu_task_info *task)
 467 {
 468         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 469                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 470                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 471 }
 472
 473 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 474 {
 475         int bits;
 476         u32 raw = 0;
 477         struct vpu_service_info *pservice = data->pservice;
 478         struct vpu_subdev_data *subdata, *n;
 479
 480         if (pservice->subcnt < 2)
 481                 return;
 482
 483         if (pservice->curr_mode == data->mode)
 484                 return;
 485
 486         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 487         list_for_each_entry_safe(subdata, n,
 488                                  &pservice->subdev_list, lnk_service) {
 489                 if (data != subdata && subdata->mmu_dev &&
 490                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 491                         clear_bit(MMU_ACTIVATED, &subdata->state);
 492                 }
 493         }
 494         bits = 1 << pservice->mode_bit;
 495 #ifdef CONFIG_MFD_SYSCON
 496         if (pservice->grf) {
 497                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 498
 499                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 500                         regmap_write(pservice->grf, pservice->mode_ctrl,
 501                                      raw | bits | (bits << 16));
 502                 else
 503                         regmap_write(pservice->grf, pservice->mode_ctrl,
 504                                      (raw & (~bits)) | (bits << 16));
 505         } else if (pservice->grf_base) {
 506                 u32 *grf_base = pservice->grf_base;
 507
 508                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 509                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 510                         writel_relaxed(raw | bits | (bits << 16),
 511                                        grf_base + pservice->mode_ctrl / 4);
 512                 else
 513                         writel_relaxed((raw & (~bits)) | (bits << 16),
 514                                        grf_base + pservice->mode_ctrl / 4);
 515         } else {
 516                 vpu_err("no grf resource define, switch decoder failed\n");
 517                 return;
 518         }
 519 #else
 520         if (pservice->grf_base) {
 521                 u32 *grf_base = pservice->grf_base;
 522
 523                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 524                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 525                         writel_relaxed(raw | bits | (bits << 16),
 526                                        grf_base + pservice->mode_ctrl / 4);
 527                 else
 528                         writel_relaxed((raw & (~bits)) | (bits << 16),
 529                                        grf_base + pservice->mode_ctrl / 4);
 530         } else {
 531                 vpu_err("no grf resource define, switch decoder failed\n");
 532                 return;
 533         }
 534 #endif
 535         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 536                 set_bit(MMU_ACTIVATED, &data->state);
 537                 if (!atomic_read(&pservice->enabled))
 538                         /* FIXME BUG_ON should not be used in mass produce */
 539                         BUG_ON(!atomic_read(&pservice->enabled));
 540         }
 541
 542         pservice->prev_mode = pservice->curr_mode;
 543         pservice->curr_mode = data->mode;
 544 }
 545
 546 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 547 {
 548         /*
 549          * In case of VPU Combo, it require HW switch its running mode
 550          * before the other HW component start work. set current HW running
 551          * mode to none, can ensure HW switch to its reqired mode properly.
 552          */
 553         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 554 }
 555
 556 static int vpu_get_clk(struct vpu_service_info *pservice)
 557 {
 558 #if VCODEC_CLOCK_ENABLE
 559         struct device *dev = pservice->dev;
 560
 561         switch (pservice->dev_id) {
 562         case VCODEC_DEVICE_ID_HEVC:
 563                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 564                 if (IS_ERR(pservice->pd_video)) {
 565                         pservice->pd_video = NULL;
 566                         dev_info(dev, "failed on clk_get pd_hevc\n");
 567                 }
 568         case VCODEC_DEVICE_ID_COMBO:
 569         case VCODEC_DEVICE_ID_RKVDEC:
 570                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 571                 if (IS_ERR(pservice->clk_cabac)) {
 572                         dev_err(dev, "failed on clk_get clk_cabac\n");
 573                         pservice->clk_cabac = NULL;
 574                 }
 575                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 576                 if (IS_ERR(pservice->clk_core)) {
 577                         dev_err(dev, "failed on clk_get clk_core\n");
 578                         pservice->clk_core = NULL;
 579                         return -1;
 580                 }
 581         case VCODEC_DEVICE_ID_VPU:
 582                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 583                 if (IS_ERR(pservice->aclk_vcodec)) {
 584                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 585                         pservice->aclk_vcodec = NULL;
 586                         return -1;
 587                 }
 588
 589                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 590                 if (IS_ERR(pservice->hclk_vcodec)) {
 591                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 592                         pservice->hclk_vcodec = NULL;
 593                         return -1;
 594                 }
 595                 if (pservice->pd_video == NULL) {
 596                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 597                         if (IS_ERR(pservice->pd_video)) {
 598                                 pservice->pd_video = NULL;
 599                                 dev_info(dev, "do not have pd_video\n");
 600                         }
 601                 }
 602                 break;
 603         default:
 604                 break;
 605         }
 606
 607         return 0;
 608 #else
 609         return 0;
 610 #endif
 611 }
 612
 613 static void _vpu_reset(struct vpu_subdev_data *data)
 614 {
 615         struct vpu_service_info *pservice = data->pservice;
 616         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 617
 618         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 619                 type = IDLE_REQ_HEVC;
 620
 621         dev_info(pservice->dev, "resetting...\n");
 622         WARN_ON(pservice->reg_codec != NULL);
 623         WARN_ON(pservice->reg_pproc != NULL);
 624         WARN_ON(pservice->reg_resev != NULL);
 625         pservice->reg_codec = NULL;
 626         pservice->reg_pproc = NULL;
 627         pservice->reg_resev = NULL;
 628
 629 #ifdef CONFIG_RESET_CONTROLLER
 630         dev_info(pservice->dev, "for 3288/3368...");
 631         if (of_machine_is_compatible("rockchip,rk3288"))
 632                 rockchip_pmu_idle_request(pservice->dev, true);
 633         if (pservice->rst_a && pservice->rst_h) {
 634                 dev_info(pservice->dev, "vpu reset in\n");
 635
 636                 if (pservice->rst_v)
 637                         reset_control_assert(pservice->rst_v);
 638                 reset_control_assert(pservice->rst_a);
 639                 reset_control_assert(pservice->rst_h);
 640                 udelay(5);
 641
 642                 reset_control_deassert(pservice->rst_h);
 643                 reset_control_deassert(pservice->rst_a);
 644                 if (pservice->rst_v)
 645                         reset_control_deassert(pservice->rst_v);
 646         } else if (pservice->rst_v) {
 647                 dev_info(pservice->dev, "hevc reset in\n");
 648                 reset_control_assert(pservice->rst_v);
 649                 udelay(5);
 650
 651                 reset_control_deassert(pservice->rst_v);
 652         }
 653         if (of_machine_is_compatible("rockchip,rk3288"))
 654                 rockchip_pmu_idle_request(pservice->dev, false);
 655 #endif
 656 }
 657
 658 static void vpu_reset(struct vpu_subdev_data *data)
 659 {
 660         struct vpu_service_info *pservice = data->pservice;
 661
 662         _vpu_reset(data);
 663         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 664                 if (atomic_read(&pservice->enabled)) {
 665                         /* Need to reset iommu */
 666                         vcodec_iommu_detach(data->iommu_info);
 667                         vcodec_iommu_attach(data->iommu_info);
 668                 } else {
 669                         /* FIXME BUG_ON should not be used in mass produce */
 670                         BUG_ON(!atomic_read(&pservice->enabled));
 671                 }
 672         }
 673
 674         atomic_set(&pservice->reset_request, 0);
 675         dev_info(pservice->dev, "reset done\n");
 676 }
 677
 678 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 679 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 680                                       struct vpu_session *session)
 681 {
 682         struct vpu_reg *reg, *n;
 683
 684         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 685                 reg_deinit(data, reg);
 686         }
 687         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 688                 reg_deinit(data, reg);
 689         }
 690         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 691                 reg_deinit(data, reg);
 692         }
 693 }
 694
 695 static void vpu_service_clear(struct vpu_subdev_data *data)
 696 {
 697         struct vpu_reg *reg, *n;
 698         struct vpu_session *session, *s;
 699         struct vpu_service_info *pservice = data->pservice;
 700
 701         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 702                 reg_deinit(data, reg);
 703         }
 704
 705         /* wake up session wait event to prevent the timeout hw reset
 706          * during reboot procedure.
 707          */
 708         list_for_each_entry_safe(session, s,
 709                                  &pservice->session, list_session)
 710                 wake_up(&session->wait);
 711 }
 712
 713 static void vpu_service_dump(struct vpu_service_info *pservice)
 714 {
 715 }
 716
 717
 718 static void vpu_service_power_off(struct vpu_service_info *pservice)
 719 {
 720         int total_running;
 721         struct vpu_subdev_data *data = NULL, *n;
 722         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 723
 724         if (!ret)
 725                 return;
 726
 727         total_running = atomic_read(&pservice->total_running);
 728         if (total_running) {
 729                 pr_alert("alert: power off when %d task running!!\n",
 730                          total_running);
 731                 mdelay(50);
 732                 pr_alert("alert: delay 50 ms for running task\n");
 733                 vpu_service_dump(pservice);
 734         }
 735
 736         dev_dbg(pservice->dev, "power off...\n");
 737
 738         udelay(5);
 739
 740         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 741                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 742                         clear_bit(MMU_ACTIVATED, &data->state);
 743                         vcodec_iommu_detach(data->iommu_info);
 744                 }
 745         }
 746         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 747         pm_runtime_put(pservice->dev);
 748 #if VCODEC_CLOCK_ENABLE
 749                 if (pservice->pd_video)
 750                         clk_disable_unprepare(pservice->pd_video);
 751                 if (pservice->hclk_vcodec)
 752                         clk_disable_unprepare(pservice->hclk_vcodec);
 753                 if (pservice->aclk_vcodec)
 754                         clk_disable_unprepare(pservice->aclk_vcodec);
 755                 if (pservice->clk_core)
 756                         clk_disable_unprepare(pservice->clk_core);
 757                 if (pservice->clk_cabac)
 758                         clk_disable_unprepare(pservice->clk_cabac);
 759 #endif
 760
 761         atomic_add(1, &pservice->power_off_cnt);
 762         wake_unlock(&pservice->wake_lock);
 763         dev_dbg(pservice->dev, "power off done\n");
 764 }
 765
 766 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 767 {
 768         queue_delayed_work(system_wq, &pservice->power_off_work,
 769                            VPU_POWER_OFF_DELAY);
 770 }
 771
 772 static void vpu_power_off_work(struct work_struct *work_s)
 773 {
 774         struct delayed_work *dlwork = container_of(work_s,
 775                         struct delayed_work, work);
 776         struct vpu_service_info *pservice = container_of(dlwork,
 777                         struct vpu_service_info, power_off_work);
 778
 779         if (mutex_trylock(&pservice->lock)) {
 780                 vpu_service_power_off(pservice);
 781                 mutex_unlock(&pservice->lock);
 782         } else {
 783                 /* Come back later if the device is busy... */
 784                 vpu_queue_power_off_work(pservice);
 785         }
 786 }
 787
 788 static void vpu_service_power_on(struct vpu_subdev_data *data,
 789                                  struct vpu_service_info *pservice)
 790 {
 791         int ret;
 792         ktime_t now = ktime_get();
 793
 794         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC ||
 795             atomic_read(&pservice->power_on_cnt)) {
 796                 /* NSEC_PER_SEC */
 797                 cancel_delayed_work_sync(&pservice->power_off_work);
 798                 vpu_queue_power_off_work(pservice);
 799                 pservice->last = now;
 800         }
 801         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 802         if (!ret) {
 803                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 804                         set_bit(MMU_ACTIVATED, &data->state);
 805                         vcodec_iommu_attach(data->iommu_info);
 806                 }
 807                 return;
 808         }
 809
 810         dev_dbg(pservice->dev, "power on\n");
 811
 812 #define BIT_VCODEC_CLK_SEL      (1<<10)
 813         if (of_machine_is_compatible("rockchip,rk3126"))
 814                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 815                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 816                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 817
 818 #if VCODEC_CLOCK_ENABLE
 819         if (pservice->aclk_vcodec)
 820                 clk_prepare_enable(pservice->aclk_vcodec);
 821         if (pservice->hclk_vcodec)
 822                 clk_prepare_enable(pservice->hclk_vcodec);
 823         if (pservice->clk_core)
 824                 clk_prepare_enable(pservice->clk_core);
 825         if (pservice->clk_cabac)
 826                 clk_prepare_enable(pservice->clk_cabac);
 827         if (pservice->pd_video)
 828                 clk_prepare_enable(pservice->pd_video);
 829 #endif
 830         pm_runtime_get_sync(pservice->dev);
 831
 832         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 833                 set_bit(MMU_ACTIVATED, &data->state);
 834                 if (atomic_read(&pservice->enabled))
 835                         vcodec_iommu_attach(data->iommu_info);
 836                 else
 837                         /*
 838                          * FIXME BUG_ON should not be used in mass
 839                          * produce.
 840                          */
 841                         BUG_ON(!atomic_read(&pservice->enabled));
 842         }
 843
 844         udelay(5);
 845         atomic_add(1, &pservice->power_on_cnt);
 846         wake_lock(&pservice->wake_lock);
 847 }
 848
 849 static inline bool reg_check_interlace(struct vpu_reg *reg)
 850 {
 851         u32 type = (reg->reg[3] & (1 << 23));
 852
 853         return (type > 0);
 854 }
 855
 856 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 857 {
 858         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 859
 860         return type;
 861 }
 862
 863 static inline int reg_probe_width(struct vpu_reg *reg)
 864 {
 865         int width_in_mb = reg->reg[4] >> 23;
 866
 867         return width_in_mb * 16;
 868 }
 869
 870 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 871 {
 872         int y_virstride = reg->reg[8];
 873
 874         return y_virstride;
 875 }
 876
 877 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 878                 struct vpu_session *session,
 879                 struct vpu_reg *reg,
 880                 int fd)
 881 {
 882         int hdl;
 883         int ret = 0;
 884         struct vcodec_mem_region *mem_region;
 885
 886         hdl = vcodec_iommu_import(data->iommu_info, session, fd);
 887         if (hdl < 0)
 888                 return hdl;
 889
 890         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 891         if (mem_region == NULL) {
 892                 vpu_err("allocate memory for iommu memory region failed\n");
 893                 vcodec_iommu_free(data->iommu_info, session, hdl);
 894                 return -ENOMEM;
 895         }
 896
 897         mem_region->hdl = hdl;
 898         ret = vcodec_iommu_map_iommu(data->iommu_info, session, mem_region->hdl,
 899                                      &mem_region->iova, &mem_region->len);
 900         if (ret < 0) {
 901                 vpu_err("fd %d ion map iommu failed\n", fd);
 902                 kfree(mem_region);
 903                 vcodec_iommu_free(data->iommu_info, session, hdl);
 904
 905                 return -EFAULT;
 906         }
 907         INIT_LIST_HEAD(&mem_region->reg_lnk);
 908         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 909         return mem_region->iova;
 910 }
 911
 912 /*
 913  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 914  * it by pps id in video stream data.
 915  *
 916  * So we need to translate the address in iommu case. The address data is also
 917  * 10bit fd + 22bit offset mode.
 918  * Because userspace decoder do not give the pps id in the register file sets
 919  * kernel driver need to translate each scaling list address in pps buffer which
 920  * means 256 pps for H.264, 64 pps for H.265.
 921  *
 922  * In order to optimize the performance kernel driver ask userspace decoder to
 923  * set all scaling list address in pps buffer to the same one which will be used
 924  * on current decoding task. Then kernel driver can only translate the first
 925  * address then copy it all pps buffer.
 926  */
 927 static int fill_scaling_list_addr_in_pps(
 928                 struct vpu_subdev_data *data,
 929                 struct vpu_reg *reg,
 930                 char *pps,
 931                 int pps_info_count,
 932                 int pps_info_size,
 933                 int scaling_list_addr_offset)
 934 {
 935         int base = scaling_list_addr_offset;
 936         int scaling_fd = 0;
 937         u32 scaling_offset;
 938
 939         scaling_offset  = (u32)pps[base + 0];
 940         scaling_offset += (u32)pps[base + 1] << 8;
 941         scaling_offset += (u32)pps[base + 2] << 16;
 942         scaling_offset += (u32)pps[base + 3] << 24;
 943
 944         scaling_fd = scaling_offset & 0x3ff;
 945         scaling_offset = scaling_offset >> 10;
 946
 947         if (scaling_fd > 0) {
 948                 int i = 0;
 949                 u32 tmp = vcodec_fd_to_iova(data, reg->session, reg,
 950                                             scaling_fd);
 951
 952                 if (IS_ERR_VALUE(tmp))
 953                         return -1;
 954                 tmp += scaling_offset;
 955
 956                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 957                         pps[base + 0] = (tmp >>  0) & 0xff;
 958                         pps[base + 1] = (tmp >>  8) & 0xff;
 959                         pps[base + 2] = (tmp >> 16) & 0xff;
 960                         pps[base + 3] = (tmp >> 24) & 0xff;
 961                 }
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data,
 968                                 struct vpu_session *session,
 969                                 const u8 *tbl,
 970                                 int size, struct vpu_reg *reg,
 971                                 struct extra_info_for_iommu *ext_inf)
 972 {
 973         struct vpu_service_info *pservice = data->pservice;
 974         struct vpu_task_info *task = reg->task;
 975         enum FORMAT_TYPE type;
 976         int hdl;
 977         int ret = 0;
 978         struct vcodec_mem_region *mem_region;
 979         int i;
 980         int offset = 0;
 981
 982         if (tbl == NULL || size <= 0) {
 983                 dev_err(pservice->dev, "input arguments invalidate\n");
 984                 return -EINVAL;
 985         }
 986
 987         if (task->get_fmt)
 988                 type = task->get_fmt(reg->reg);
 989         else {
 990                 dev_err(pservice->dev, "invalid task with NULL get_fmt\n");
 991                 return -EINVAL;
 992         }
 993
 994         for (i = 0; i < size; i++) {
 995                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
 996
 997                 /* if userspace do not set the fd at this register, skip */
 998                 if (usr_fd == 0)
 999                         continue;
1000
1001                 /*
1002                  * for avoiding cache sync issue, we need to map/unmap
1003                  * input buffer every time. FIX ME, if it is unnecessary
1004                  */
1005                 if (task->reg_rlc == tbl[i])
1006                         vcodec_iommu_free_fd(data->iommu_info, session, usr_fd);
1007                 /*
1008                  * special offset scale case
1009                  *
1010                  * This translation is for fd + offset translation.
1011                  * One register has 32bits. We need to transfer both buffer file
1012                  * handle and the start address offset so we packet file handle
1013                  * and offset together using below format.
1014                  *
1015                  *  0~9  bit for buffer file handle range 0 ~ 1023
1016                  * 10~31 bit for offset range 0 ~ 4M
1017                  *
1018                  * But on 4K case the offset can be larger the 4M
1019                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1020                  * But MPEG4 will use the same register for colmv and it do not
1021                  * need scale.
1022                  *
1023                  * RKVdec do not have this issue.
1024                  */
1025                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1026                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1027                         offset = reg->reg[tbl[i]] >> 10 << 4;
1028                 else
1029                         offset = reg->reg[tbl[i]] >> 10;
1030
1031                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d i %d\n",
1032                           tbl[i], usr_fd, offset, i);
1033
1034                 hdl = vcodec_iommu_import(data->iommu_info, session, usr_fd);
1035
1036                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1037                         int pps_info_offset;
1038                         int pps_info_count;
1039                         int pps_info_size;
1040                         int scaling_list_addr_offset;
1041
1042                         switch (type) {
1043                         case FMT_H264D: {
1044                                 pps_info_offset = offset;
1045                                 pps_info_count = 256;
1046                                 pps_info_size = 32;
1047                                 scaling_list_addr_offset = 23;
1048                         } break;
1049                         case FMT_H265D: {
1050                                 pps_info_offset = 0;
1051                                 pps_info_count = 64;
1052                                 pps_info_size = 80;
1053                                 scaling_list_addr_offset = 74;
1054                         } break;
1055                         default: {
1056                                 pps_info_offset = 0;
1057                                 pps_info_count = 0;
1058                                 pps_info_size = 0;
1059                                 scaling_list_addr_offset = 0;
1060                         } break;
1061                         }
1062
1063                         vpu_debug(DEBUG_PPS_FILL,
1064                                   "scaling list filling parameter:\n");
1065                         vpu_debug(DEBUG_PPS_FILL,
1066                                   "pps_info_offset %d\n", pps_info_offset);
1067                         vpu_debug(DEBUG_PPS_FILL,
1068                                   "pps_info_count  %d\n", pps_info_count);
1069                         vpu_debug(DEBUG_PPS_FILL,
1070                                   "pps_info_size   %d\n", pps_info_size);
1071                         vpu_debug(DEBUG_PPS_FILL,
1072                                   "scaling_list_addr_offset %d\n",
1073                                   scaling_list_addr_offset);
1074
1075                         if (pps_info_count) {
1076                                 u8 *pps;
1077
1078                                 mutex_lock(&pservice->lock);
1079
1080                                 pps = vcodec_iommu_map_kernel
1081                                         (data->iommu_info, session, hdl);
1082
1083                                 vpu_debug(DEBUG_PPS_FILL,
1084                                           "scaling list setting pps %p\n", pps);
1085                                 pps += pps_info_offset;
1086
1087                                 fill_scaling_list_addr_in_pps
1088                                         (data, reg, pps, pps_info_count,
1089                                          pps_info_size,
1090                                          scaling_list_addr_offset);
1091
1092                                 vcodec_iommu_unmap_kernel
1093                                         (data->iommu_info, session, hdl);
1094                                 mutex_unlock(&pservice->lock);
1095                         }
1096                 }
1097
1098                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1099
1100                 if (!mem_region) {
1101                         vcodec_iommu_free(data->iommu_info, session, hdl);
1102                         return -ENOMEM;
1103                 }
1104
1105                 mem_region->hdl = hdl;
1106                 mem_region->reg_idx = tbl[i];
1107
1108                 ret = vcodec_iommu_map_iommu(data->iommu_info, session,
1109                                              mem_region->hdl, &mem_region->iova,
1110                                              &mem_region->len);
1111                 if (ret < 0) {
1112                         dev_err(pservice->dev,
1113                                 "reg %d fd %d ion map iommu failed\n",
1114                                 tbl[i], usr_fd);
1115                         kfree(mem_region);
1116                         vcodec_iommu_free(data->iommu_info, session, hdl);
1117                         return ret;
1118                 }
1119
1120                 /*
1121                  * special for vpu dec num 12: record decoded length
1122                  * hacking for decoded length
1123                  * NOTE: not a perfect fix, the fd is not recorded
1124                  */
1125                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1126                         reg->dec_base = mem_region->iova + offset;
1127                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1128                                   reg->dec_base);
1129                 }
1130
1131                 reg->reg[tbl[i]] = mem_region->iova + offset;
1132                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1133                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1134         }
1135
1136         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1137                 for (i = 0; i < ext_inf->cnt; i++) {
1138                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1139                                   ext_inf->elem[i].index,
1140                                   ext_inf->elem[i].offset);
1141                         reg->reg[ext_inf->elem[i].index] +=
1142                                 ext_inf->elem[i].offset;
1143                 }
1144         }
1145
1146         return 0;
1147 }
1148
1149 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1150                                         struct vpu_session *session,
1151                                         struct vpu_reg *reg,
1152                                         struct extra_info_for_iommu *ext_inf)
1153 {
1154         struct vpu_service_info *pservice = data->pservice;
1155         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1156
1157         if (type < FMT_TYPE_BUTT) {
1158                 const struct vpu_trans_info *info = &reg->trans[type];
1159                 const u8 *tbl = info->table;
1160                 int size = info->count;
1161
1162                 return vcodec_bufid_to_iova(data, session, tbl, size, reg,
1163                                             ext_inf);
1164         }
1165
1166         dev_err(pservice->dev, "found invalid format type!\n");
1167         return -EINVAL;
1168 }
1169
1170 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1171 {
1172
1173         if (!of_machine_is_compatible("rockchip,rk2928g")) {
1174                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1175                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1176                                 if (reg_probe_width(reg) > 3200) {
1177                                         /*raise frequency for 4k avc.*/
1178                                         reg->freq = VPU_FREQ_600M;
1179                                 }
1180                         } else {
1181                                 if (reg_check_interlace(reg))
1182                                         reg->freq = VPU_FREQ_400M;
1183                         }
1184                 }
1185                 if (data->hw_id == HEVC_ID) {
1186                         if (reg_probe_hevc_y_stride(reg) > 60000)
1187                                 reg->freq = VPU_FREQ_400M;
1188                 }
1189                 if (reg->type == VPU_PP)
1190                         reg->freq = VPU_FREQ_400M;
1191         }
1192 }
1193
1194 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1195                                 struct vpu_session *session,
1196                                 void __user *src, u32 size)
1197 {
1198         struct vpu_service_info *pservice = data->pservice;
1199         int extra_size = 0;
1200         struct extra_info_for_iommu extra_info;
1201         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1202                                       GFP_KERNEL);
1203
1204         vpu_debug_enter();
1205
1206         if (!reg) {
1207                 vpu_err("error: kzalloc failed\n");
1208                 return NULL;
1209         }
1210
1211         if (size > data->reg_size) {
1212                 extra_size = size - data->reg_size;
1213                 size = data->reg_size;
1214         }
1215         reg->session = session;
1216         reg->data = data;
1217         reg->type = session->type;
1218         reg->size = size;
1219         reg->freq = VPU_FREQ_DEFAULT;
1220         reg->task = &data->task_info[session->type];
1221         reg->trans = data->trans_info;
1222         reg->reg = (u32 *)&reg[1];
1223         INIT_LIST_HEAD(&reg->session_link);
1224         INIT_LIST_HEAD(&reg->status_link);
1225
1226         INIT_LIST_HEAD(&reg->mem_region_list);
1227
1228         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1229                 vpu_err("error: copy_from_user failed\n");
1230                 kfree(reg);
1231                 return NULL;
1232         }
1233
1234         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1235                 vpu_err("error: copy_from_user failed\n");
1236                 kfree(reg);
1237                 return NULL;
1238         }
1239
1240         if (vcodec_reg_address_translate(data, session, reg, &extra_info) < 0) {
1241                 int i = 0;
1242
1243                 vpu_err("error: translate reg address failed, dumping regs\n");
1244                 for (i = 0; i < size >> 2; i++)
1245                         dev_err(pservice->dev, "reg[%02d]: %08x\n",
1246                                 i, *((u32 *)src + i));
1247
1248                 kfree(reg);
1249                 return NULL;
1250         }
1251
1252         mutex_lock(&pservice->lock);
1253         list_add_tail(&reg->status_link, &pservice->waiting);
1254         list_add_tail(&reg->session_link, &session->waiting);
1255         mutex_unlock(&pservice->lock);
1256
1257         if (pservice->auto_freq)
1258                 get_reg_freq(data, reg);
1259
1260         vpu_debug_leave();
1261
1262         return reg;
1263 }
1264
1265 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1266 {
1267         struct vpu_service_info *pservice = data->pservice;
1268         struct vcodec_mem_region *mem_region = NULL, *n;
1269
1270         list_del_init(&reg->session_link);
1271         list_del_init(&reg->status_link);
1272         if (reg == pservice->reg_codec)
1273                 pservice->reg_codec = NULL;
1274         if (reg == pservice->reg_pproc)
1275                 pservice->reg_pproc = NULL;
1276
1277         /* release memory region attach to this registers table. */
1278         list_for_each_entry_safe(mem_region, n,
1279                         &reg->mem_region_list, reg_lnk) {
1280                 vcodec_iommu_unmap_iommu(data->iommu_info, reg->session,
1281                                          mem_region->hdl);
1282                 vcodec_iommu_free(data->iommu_info, reg->session,
1283                                   mem_region->hdl);
1284                 list_del_init(&mem_region->reg_lnk);
1285                 kfree(mem_region);
1286         }
1287
1288         kfree(reg);
1289 }
1290
1291 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1292                                  struct vpu_reg *reg)
1293 {
1294         vpu_debug_enter();
1295         list_del_init(&reg->status_link);
1296         list_add_tail(&reg->status_link, &pservice->running);
1297
1298         list_del_init(&reg->session_link);
1299         list_add_tail(&reg->session_link, &reg->session->running);
1300         vpu_debug_leave();
1301 }
1302
1303 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1304 {
1305         int i;
1306         u32 *dst = reg->reg;
1307
1308         vpu_debug_enter();
1309         for (i = 0; i < count; i++, src++)
1310                 *dst++ = readl_relaxed(src);
1311
1312         dst = (u32 *)&reg->reg[0];
1313         for (i = 0; i < count; i++)
1314                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1315
1316         vpu_debug_leave();
1317 }
1318
1319 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1320                                  struct vpu_reg *reg)
1321 {
1322         struct vpu_service_info *pservice = data->pservice;
1323         struct vpu_hw_info *hw_info = data->hw_info;
1324         struct vpu_task_info *task = reg->task;
1325
1326         vpu_debug_enter();
1327
1328         list_del_init(&reg->status_link);
1329         list_add_tail(&reg->status_link, &pservice->done);
1330
1331         list_del_init(&reg->session_link);
1332         list_add_tail(&reg->session_link, &reg->session->done);
1333
1334         switch (reg->type) {
1335         case VPU_ENC: {
1336                 pservice->reg_codec = NULL;
1337                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1338                 reg->reg[task->reg_irq] = pservice->irq_status;
1339         } break;
1340         case VPU_DEC: {
1341                 pservice->reg_codec = NULL;
1342                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1343
1344                 /* revert hack for decoded length */
1345                 if (task->reg_len > 0) {
1346                         int reg_len = task->reg_len;
1347                         u32 dec_get = reg->reg[reg_len];
1348                         s32 dec_length = dec_get - reg->dec_base;
1349
1350                         vpu_debug(DEBUG_REGISTER,
1351                                   "dec_get %08x dec_length %d\n",
1352                                   dec_get, dec_length);
1353                         reg->reg[reg_len] = dec_length << 10;
1354                 }
1355
1356                 reg->reg[task->reg_irq] = pservice->irq_status;
1357         } break;
1358         case VPU_PP: {
1359                 pservice->reg_pproc = NULL;
1360                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1361                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1362         } break;
1363         case VPU_DEC_PP: {
1364                 u32 pipe_mode;
1365                 u32 *regs = data->dec_dev.regs;
1366
1367                 pservice->reg_codec = NULL;
1368                 pservice->reg_pproc = NULL;
1369
1370                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1371
1372                 /* NOTE: remove pp pipeline mode flag first */
1373                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1374                 pipe_mode &= ~task->pipe_mask;
1375                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1376
1377                 /* revert hack for decoded length */
1378                 if (task->reg_len > 0) {
1379                         int reg_len = task->reg_len;
1380                         u32 dec_get = reg->reg[reg_len];
1381                         s32 dec_length = dec_get - reg->dec_base;
1382
1383                         vpu_debug(DEBUG_REGISTER,
1384                                   "dec_get %08x dec_length %d\n",
1385                                   dec_get, dec_length);
1386                         reg->reg[reg_len] = dec_length << 10;
1387                 }
1388
1389                 reg->reg[task->reg_irq] = pservice->irq_status;
1390         } break;
1391         default: {
1392                 vpu_err("error: copy reg from hw with unknown type %d\n",
1393                         reg->type);
1394         } break;
1395         }
1396         vcodec_exit_mode(data);
1397
1398         atomic_sub(1, &reg->session->task_running);
1399         atomic_sub(1, &pservice->total_running);
1400         wake_up(&reg->session->wait);
1401
1402         vpu_debug_leave();
1403 }
1404
1405 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1406                                  struct vpu_reg *reg)
1407 {
1408         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1409
1410         if (curr == reg->freq)
1411                 return;
1412
1413         atomic_set(&pservice->freq_status, reg->freq);
1414         switch (reg->freq) {
1415         case VPU_FREQ_200M: {
1416                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1417         } break;
1418         case VPU_FREQ_266M: {
1419                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1420         } break;
1421         case VPU_FREQ_300M: {
1422                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1423         } break;
1424         case VPU_FREQ_400M: {
1425                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1426         } break;
1427         case VPU_FREQ_500M: {
1428                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1429         } break;
1430         case VPU_FREQ_600M: {
1431                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1432         } break;
1433         default: {
1434                 unsigned long rate = 300*MHZ;
1435
1436                 if (of_machine_is_compatible("rockchip,rk2928g"))
1437                         rate = 400*MHZ;
1438
1439                 clk_set_rate(pservice->aclk_vcodec, rate);
1440         } break;
1441         }
1442 }
1443
1444 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1445 {
1446         struct vpu_service_info *pservice = data->pservice;
1447         struct vpu_task_info *task = reg->task;
1448         struct vpu_hw_info *hw_info = data->hw_info;
1449         int i;
1450         u32 *src = (u32 *)&reg->reg[0];
1451         u32 enable_mask = task->enable_mask;
1452         u32 gating_mask = task->gating_mask;
1453         u32 reg_en = task->reg_en;
1454
1455         vpu_debug_enter();
1456
1457         atomic_add(1, &pservice->total_running);
1458         atomic_add(1, &reg->session->task_running);
1459
1460         if (pservice->auto_freq)
1461                 vpu_service_set_freq(pservice, reg);
1462
1463         vcodec_enter_mode(data);
1464
1465         switch (reg->type) {
1466         case VPU_ENC: {
1467                 u32 *dst = data->enc_dev.regs;
1468                 u32 base = 0;
1469                 u32 end  = hw_info->enc_reg_num;
1470                 /* u32 reg_gating = task->reg_gating; */
1471
1472                 pservice->reg_codec = reg;
1473
1474                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1475                           base, end, reg_en, enable_mask, gating_mask);
1476
1477                 VEPU_CLEAN_CACHE(dst);
1478
1479                 if (debug & DEBUG_SET_REG)
1480                         for (i = base; i < end; i++)
1481                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1482                                           i, src[i]);
1483
1484                 /*
1485                  * NOTE: encoder need to setup mode first
1486                  */
1487                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1488
1489                 /* NOTE: encoder gating is not on enable register */
1490                 /* src[reg_gating] |= gating_mask; */
1491
1492                 for (i = base; i < end; i++) {
1493                         if (i != reg_en)
1494                                 writel_relaxed(src[i], dst + i);
1495                 }
1496
1497                 writel(src[reg_en], dst + reg_en);
1498                 dsb(sy);
1499
1500                 time_record(reg->task, 0);
1501         } break;
1502         case VPU_DEC: {
1503                 u32 *dst = data->dec_dev.regs;
1504                 u32 len = hw_info->dec_reg_num;
1505                 u32 base = hw_info->base_dec;
1506                 u32 end  = hw_info->end_dec;
1507
1508                 pservice->reg_codec = reg;
1509
1510                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1511                           base, end, reg_en, enable_mask, gating_mask);
1512
1513                 VDPU_CLEAN_CACHE(dst);
1514
1515                 /* on rkvdec set cache size to 64byte */
1516                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1517                         u32 *cache_base = dst + 0x100;
1518                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1519                         writel_relaxed(val, cache_base + 0x07);
1520                         writel_relaxed(val, cache_base + 0x17);
1521                 }
1522
1523                 if (debug & DEBUG_SET_REG)
1524                         for (i = 0; i < len; i++)
1525                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1526                                           i, src[i]);
1527                 /*
1528                  * NOTE: The end register is invalid. Do NOT write to it
1529                  *       Also the base register must be written
1530                  */
1531                 for (i = base; i < end; i++) {
1532                         if (i != reg_en)
1533                                 writel_relaxed(src[i], dst + i);
1534                 }
1535
1536                 writel(src[reg_en] | gating_mask, dst + reg_en);
1537                 dsb(sy);
1538
1539                 time_record(reg->task, 0);
1540         } break;
1541         case VPU_PP: {
1542                 u32 *dst = data->dec_dev.regs;
1543                 u32 base = hw_info->base_pp;
1544                 u32 end  = hw_info->end_pp;
1545
1546                 pservice->reg_pproc = reg;
1547
1548                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1549                           base, end, reg_en, enable_mask, gating_mask);
1550
1551                 if (debug & DEBUG_SET_REG)
1552                         for (i = base; i < end; i++)
1553                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1554                                           i, src[i]);
1555
1556                 for (i = base; i < end; i++) {
1557                         if (i != reg_en)
1558                                 writel_relaxed(src[i], dst + i);
1559                 }
1560
1561                 writel(src[reg_en] | gating_mask, dst + reg_en);
1562                 dsb(sy);
1563
1564                 time_record(reg->task, 0);
1565         } break;
1566         case VPU_DEC_PP: {
1567                 u32 *dst = data->dec_dev.regs;
1568                 u32 base = hw_info->base_dec_pp;
1569                 u32 end  = hw_info->end_dec_pp;
1570
1571                 pservice->reg_codec = reg;
1572                 pservice->reg_pproc = reg;
1573
1574                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1575                           base, end, reg_en, enable_mask, gating_mask);
1576
1577                 /* VDPU_SOFT_RESET(dst); */
1578                 VDPU_CLEAN_CACHE(dst);
1579
1580                 if (debug & DEBUG_SET_REG)
1581                         for (i = base; i < end; i++)
1582                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1583                                           i, src[i]);
1584
1585                 for (i = base; i < end; i++) {
1586                         if (i != reg_en)
1587                                 writel_relaxed(src[i], dst + i);
1588                 }
1589
1590                 /* NOTE: dec output must be disabled */
1591
1592                 writel(src[reg_en] | gating_mask, dst + reg_en);
1593                 dsb(sy);
1594
1595                 time_record(reg->task, 0);
1596         } break;
1597         default: {
1598                 vpu_err("error: unsupport session type %d", reg->type);
1599                 atomic_sub(1, &pservice->total_running);
1600                 atomic_sub(1, &reg->session->task_running);
1601         } break;
1602         }
1603
1604         vpu_debug_leave();
1605 }
1606
1607 static void try_set_reg(struct vpu_subdev_data *data)
1608 {
1609         struct vpu_service_info *pservice = data->pservice;
1610
1611         vpu_debug_enter();
1612
1613         mutex_lock(&pservice->shutdown_lock);
1614         if (atomic_read(&pservice->service_on) == 0) {
1615                 mutex_unlock(&pservice->shutdown_lock);
1616                 return;
1617         }
1618         if (!list_empty(&pservice->waiting)) {
1619                 struct vpu_reg *reg_codec = pservice->reg_codec;
1620                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1621                 int can_set = 0;
1622                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1623                 int reset_request = atomic_read(&pservice->reset_request);
1624                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1625                                 struct vpu_reg, status_link);
1626
1627                 if (change_able || !reset_request) {
1628                         switch (reg->type) {
1629                         case VPU_ENC: {
1630                                 if (change_able)
1631                                         can_set = 1;
1632                         } break;
1633                         case VPU_DEC: {
1634                                 if (reg_codec == NULL)
1635                                         can_set = 1;
1636                                 if (pservice->auto_freq && (reg_pproc != NULL))
1637                                         can_set = 0;
1638                         } break;
1639                         case VPU_PP: {
1640                                 if (reg_codec == NULL) {
1641                                         if (reg_pproc == NULL)
1642                                                 can_set = 1;
1643                                 } else {
1644                                         if ((reg_codec->type == VPU_DEC) &&
1645                                             (reg_pproc == NULL))
1646                                                 can_set = 1;
1647
1648                                         /*
1649                                          * NOTE:
1650                                          * can not charge frequency
1651                                          * when vpu is working
1652                                          */
1653                                         if (pservice->auto_freq)
1654                                                 can_set = 0;
1655                                 }
1656                         } break;
1657                         case VPU_DEC_PP: {
1658                                 if (change_able)
1659                                         can_set = 1;
1660                                 } break;
1661                         default: {
1662                                 dev_err(pservice->dev,
1663                                         "undefined reg type %d\n",
1664                                         reg->type);
1665                         } break;
1666                         }
1667                 }
1668
1669                 /* then check reset request */
1670                 if (reset_request && !change_able)
1671                         reset_request = 0;
1672
1673                 /* do reset before setting registers */
1674                 if (reset_request)
1675                         vpu_reset(data);
1676
1677                 if (can_set) {
1678                         reg_from_wait_to_run(pservice, reg);
1679                         reg_copy_to_hw(reg->data, reg);
1680                 }
1681         }
1682
1683         mutex_unlock(&pservice->shutdown_lock);
1684         vpu_debug_leave();
1685 }
1686
1687 static int return_reg(struct vpu_subdev_data *data,
1688                       struct vpu_reg *reg, u32 __user *dst)
1689 {
1690         struct vpu_hw_info *hw_info = data->hw_info;
1691         size_t size = reg->size;
1692         u32 base;
1693
1694         vpu_debug_enter();
1695         switch (reg->type) {
1696         case VPU_ENC: {
1697                 base = 0;
1698         } break;
1699         case VPU_DEC: {
1700                 base = hw_info->base_dec_pp;
1701         } break;
1702         case VPU_PP: {
1703                 base = hw_info->base_pp;
1704         } break;
1705         case VPU_DEC_PP: {
1706                 base = hw_info->base_dec_pp;
1707         } break;
1708         default: {
1709                 vpu_err("error: copy reg to user with unknown type %d\n",
1710                         reg->type);
1711                 return -EFAULT;
1712         } break;
1713         }
1714
1715         if (copy_to_user(dst, &reg->reg[base], size)) {
1716                 vpu_err("error: copy_to_user failed\n");
1717                 return -EFAULT;
1718         }
1719
1720         reg_deinit(data, reg);
1721         vpu_debug_leave();
1722         return 0;
1723 }
1724
1725 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1726                               unsigned long arg)
1727 {
1728         struct vpu_subdev_data *data =
1729                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1730                              struct vpu_subdev_data, cdev);
1731         struct vpu_service_info *pservice = data->pservice;
1732         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1733
1734         vpu_debug_enter();
1735         if (NULL == session)
1736                 return -EINVAL;
1737
1738         switch (cmd) {
1739         case VPU_IOC_SET_CLIENT_TYPE: {
1740                 session->type = (enum VPU_CLIENT_TYPE)arg;
1741                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1742                           session->pid, session->type);
1743         } break;
1744         case VPU_IOC_GET_HW_FUSE_STATUS: {
1745                 struct vpu_request req;
1746
1747                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1748                           session->pid, session->type);
1749                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1750                         vpu_err("error: get hw status copy_from_user failed\n");
1751                         return -EFAULT;
1752                 } else {
1753                         void *config = (session->type != VPU_ENC) ?
1754                                        ((void *)&pservice->dec_config) :
1755                                        ((void *)&pservice->enc_config);
1756                         size_t size = (session->type != VPU_ENC) ?
1757                                       (sizeof(struct vpu_dec_config)) :
1758                                       (sizeof(struct vpu_enc_config));
1759                         if (copy_to_user((void __user *)req.req,
1760                                          config, size)) {
1761                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1762                                         session->type);
1763                                 return -EFAULT;
1764                         }
1765                 }
1766         } break;
1767         case VPU_IOC_SET_REG: {
1768                 struct vpu_request req;
1769                 struct vpu_reg *reg;
1770
1771                 vpu_service_power_on(data, pservice);
1772
1773                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1774                           session->pid, session->type);
1775                 if (copy_from_user(&req, (void __user *)arg,
1776                                    sizeof(struct vpu_request))) {
1777                         vpu_err("error: set reg copy_from_user failed\n");
1778                         return -EFAULT;
1779                 }
1780
1781                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1782                 if (NULL == reg) {
1783                         return -EFAULT;
1784                 } else {
1785                         mutex_lock(&pservice->lock);
1786                         try_set_reg(data);
1787                         mutex_unlock(&pservice->lock);
1788                 }
1789         } break;
1790         case VPU_IOC_GET_REG: {
1791                 struct vpu_request req;
1792                 struct vpu_reg *reg;
1793                 int ret;
1794
1795                 vpu_service_power_on(data, pservice);
1796
1797                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1798                           session->pid, session->type);
1799                 if (copy_from_user(&req, (void __user *)arg,
1800                                    sizeof(struct vpu_request))) {
1801                         vpu_err("error: get reg copy_from_user failed\n");
1802                         return -EFAULT;
1803                 }
1804
1805                 ret = wait_event_timeout(session->wait,
1806                                          !list_empty(&session->done),
1807                                          VPU_TIMEOUT_DELAY);
1808
1809                 if (!list_empty(&session->done)) {
1810                         if (ret < 0)
1811                                 vpu_err("warning: pid %d wait task error ret %d\n",
1812                                         session->pid, ret);
1813                         ret = 0;
1814                 } else {
1815                         if (unlikely(ret < 0)) {
1816                                 vpu_err("error: pid %d wait task ret %d\n",
1817                                         session->pid, ret);
1818                         } else if (ret == 0) {
1819                                 vpu_err("error: pid %d wait %d task done timeout\n",
1820                                         session->pid,
1821                                         atomic_read(&session->task_running));
1822                                 ret = -ETIMEDOUT;
1823                         }
1824                 }
1825
1826                 if (ret < 0) {
1827                         int task_running = atomic_read(&session->task_running);
1828
1829                         mutex_lock(&pservice->lock);
1830                         vpu_service_dump(pservice);
1831                         if (task_running) {
1832                                 atomic_set(&session->task_running, 0);
1833                                 atomic_sub(task_running,
1834                                            &pservice->total_running);
1835                                 dev_err(pservice->dev,
1836                                         "%d task is running but not return, reset hardware...",
1837                                        task_running);
1838                                 vpu_reset(data);
1839                                 dev_err(pservice->dev, "done\n");
1840                         }
1841                         vpu_service_session_clear(data, session);
1842                         mutex_unlock(&pservice->lock);
1843                         return ret;
1844                 }
1845
1846                 mutex_lock(&pservice->lock);
1847                 reg = list_entry(session->done.next,
1848                                  struct vpu_reg, session_link);
1849                 return_reg(data, reg, (u32 __user *)req.req);
1850                 mutex_unlock(&pservice->lock);
1851         } break;
1852         case VPU_IOC_PROBE_IOMMU_STATUS: {
1853                 int iommu_enable = 1;
1854
1855                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1856
1857                 if (copy_to_user((void __user *)arg,
1858                                  &iommu_enable, sizeof(int))) {
1859                         vpu_err("error: iommu status copy_to_user failed\n");
1860                         return -EFAULT;
1861                 }
1862         } break;
1863         default: {
1864                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1865         } break;
1866         }
1867         vpu_debug_leave();
1868         return 0;
1869 }
1870
1871 #ifdef CONFIG_COMPAT
1872 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1873                                      unsigned long arg)
1874 {
1875         struct vpu_subdev_data *data =
1876                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1877                              struct vpu_subdev_data, cdev);
1878         struct vpu_service_info *pservice = data->pservice;
1879         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1880
1881         vpu_debug_enter();
1882         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1883                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1884         if (NULL == session)
1885                 return -EINVAL;
1886
1887         switch (cmd) {
1888         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1889                 session->type = (enum VPU_CLIENT_TYPE)arg;
1890                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1891                           session->type);
1892         } break;
1893         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1894                 struct compat_vpu_request req;
1895
1896                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1897                           session->type);
1898                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1899                                    sizeof(struct compat_vpu_request))) {
1900                         vpu_err("error: compat get hw status copy_from_user failed\n");
1901                         return -EFAULT;
1902                 } else {
1903                         void *config = (session->type != VPU_ENC) ?
1904                                        ((void *)&pservice->dec_config) :
1905                                        ((void *)&pservice->enc_config);
1906                         size_t size = (session->type != VPU_ENC) ?
1907                                       (sizeof(struct vpu_dec_config)) :
1908                                       (sizeof(struct vpu_enc_config));
1909
1910                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1911                                          config, size)) {
1912                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1913                                         session->type);
1914                                 return -EFAULT;
1915                         }
1916                 }
1917         } break;
1918         case COMPAT_VPU_IOC_SET_REG: {
1919                 struct compat_vpu_request req;
1920                 struct vpu_reg *reg;
1921
1922                 vpu_service_power_on(data, pservice);
1923
1924                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1925                           session->type);
1926                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1927                                    sizeof(struct compat_vpu_request))) {
1928                         vpu_err("compat set_reg copy_from_user failed\n");
1929                         return -EFAULT;
1930                 }
1931                 reg = reg_init(data, session,
1932                                compat_ptr((compat_uptr_t)req.req), req.size);
1933                 if (NULL == reg) {
1934                         return -EFAULT;
1935                 } else {
1936                         mutex_lock(&pservice->lock);
1937                         try_set_reg(data);
1938                         mutex_unlock(&pservice->lock);
1939                 }
1940         } break;
1941         case COMPAT_VPU_IOC_GET_REG: {
1942                 struct compat_vpu_request req;
1943                 struct vpu_reg *reg;
1944                 int ret;
1945
1946                 vpu_service_power_on(data, pservice);
1947
1948                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1949                           session->type);
1950                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1951                                    sizeof(struct compat_vpu_request))) {
1952                         vpu_err("compat get reg copy_from_user failed\n");
1953                         return -EFAULT;
1954                 }
1955
1956                 ret = wait_event_timeout(session->wait,
1957                                          !list_empty(&session->done),
1958                                          VPU_TIMEOUT_DELAY);
1959
1960                 if (!list_empty(&session->done)) {
1961                         if (ret < 0)
1962                                 vpu_err("warning: pid %d wait task error ret %d\n",
1963                                         session->pid, ret);
1964                         ret = 0;
1965                 } else {
1966                         if (unlikely(ret < 0)) {
1967                                 vpu_err("error: pid %d wait task ret %d\n",
1968                                         session->pid, ret);
1969                         } else if (ret == 0) {
1970                                 vpu_err("error: pid %d wait %d task done timeout\n",
1971                                         session->pid,
1972                                         atomic_read(&session->task_running));
1973                                 ret = -ETIMEDOUT;
1974                         }
1975                 }
1976
1977                 if (ret < 0) {
1978                         int task_running = atomic_read(&session->task_running);
1979
1980                         mutex_lock(&pservice->lock);
1981                         vpu_service_dump(pservice);
1982                         if (task_running) {
1983                                 atomic_set(&session->task_running, 0);
1984                                 atomic_sub(task_running,
1985                                            &pservice->total_running);
1986                                 dev_err(pservice->dev,
1987                                         "%d task is running but not return, reset hardware...",
1988                                         task_running);
1989                                 vpu_reset(data);
1990                                 dev_err(pservice->dev, "done\n");
1991                         }
1992                         vpu_service_session_clear(data, session);
1993                         mutex_unlock(&pservice->lock);
1994                         return ret;
1995                 }
1996
1997                 mutex_lock(&pservice->lock);
1998                 reg = list_entry(session->done.next,
1999                                  struct vpu_reg, session_link);
2000                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
2001                 mutex_unlock(&pservice->lock);
2002         } break;
2003         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
2004                 int iommu_enable = 1;
2005
2006                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
2007
2008                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
2009                                  &iommu_enable, sizeof(int))) {
2010                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2011                         return -EFAULT;
2012                 }
2013         } break;
2014         default: {
2015                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2016         } break;
2017         }
2018         vpu_debug_leave();
2019         return 0;
2020 }
2021 #endif
2022
2023 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2024 {
2025         struct vpu_service_info *pservice = data->pservice;
2026         int ret = -EINVAL, i = 0;
2027         u32 hw_id = readl_relaxed(data->regs);
2028
2029         hw_id = (hw_id >> 16) & 0xFFFF;
2030         dev_info(pservice->dev, "checking hw id %x\n", hw_id);
2031         data->hw_info = NULL;
2032
2033         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2034                 const struct vcodec_info *info = &vcodec_info_set[i];
2035
2036                 if (hw_id == info->hw_id) {
2037                         data->hw_id = info->hw_id;
2038                         data->hw_info = info->hw_info;
2039                         data->task_info = info->task_info;
2040                         data->trans_info = info->trans_info;
2041                         ret = 0;
2042                         break;
2043                 }
2044         }
2045         return ret;
2046 }
2047
2048 static int vpu_service_open(struct inode *inode, struct file *filp)
2049 {
2050         struct vpu_subdev_data *data = container_of(
2051                         inode->i_cdev, struct vpu_subdev_data, cdev);
2052         struct vpu_service_info *pservice = data->pservice;
2053         struct vpu_session *session = NULL;
2054
2055         vpu_debug_enter();
2056
2057         session = kzalloc(sizeof(*session), GFP_KERNEL);
2058         if (!session) {
2059                 vpu_err("error: unable to allocate memory for vpu_session.");
2060                 return -ENOMEM;
2061         }
2062
2063         data->iommu_info->debug_level = debug;
2064
2065         session->type   = VPU_TYPE_BUTT;
2066         session->pid    = current->pid;
2067         INIT_LIST_HEAD(&session->waiting);
2068         INIT_LIST_HEAD(&session->running);
2069         INIT_LIST_HEAD(&session->done);
2070         INIT_LIST_HEAD(&session->list_session);
2071         init_waitqueue_head(&session->wait);
2072         atomic_set(&session->task_running, 0);
2073         mutex_lock(&pservice->lock);
2074         list_add_tail(&session->list_session, &pservice->session);
2075         filp->private_data = (void *)session;
2076         mutex_unlock(&pservice->lock);
2077
2078         dev_dbg(pservice->dev, "dev opened\n");
2079         vpu_debug_leave();
2080         return nonseekable_open(inode, filp);
2081 }
2082
2083 static int vpu_service_release(struct inode *inode, struct file *filp)
2084 {
2085         struct vpu_subdev_data *data = container_of(
2086                         inode->i_cdev, struct vpu_subdev_data, cdev);
2087         struct vpu_service_info *pservice = data->pservice;
2088         int task_running;
2089         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2090
2091         vpu_debug_enter();
2092         if (NULL == session)
2093                 return -EINVAL;
2094
2095         task_running = atomic_read(&session->task_running);
2096         if (task_running) {
2097                 dev_err(pservice->dev,
2098                         "error: session %d still has %d task running when closing\n",
2099                         session->pid, task_running);
2100                 msleep(50);
2101         }
2102         wake_up(&session->wait);
2103
2104         vpu_service_power_on(data, pservice);
2105         mutex_lock(&pservice->lock);
2106         /* remove this filp from the asynchronusly notified filp's */
2107         list_del_init(&session->list_session);
2108         vpu_service_session_clear(data, session);
2109         vcodec_iommu_clear(data->iommu_info, session);
2110         kfree(session);
2111         filp->private_data = NULL;
2112         mutex_unlock(&pservice->lock);
2113
2114         dev_info(pservice->dev, "closed\n");
2115         vpu_debug_leave();
2116         return 0;
2117 }
2118
2119 static const struct file_operations vpu_service_fops = {
2120         .unlocked_ioctl = vpu_service_ioctl,
2121         .open           = vpu_service_open,
2122         .release        = vpu_service_release,
2123 #ifdef CONFIG_COMPAT
2124         .compat_ioctl   = compat_vpu_service_ioctl,
2125 #endif
2126 };
2127
2128 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2129 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2130 static irqreturn_t vepu_irq(int irq, void *dev_id);
2131 static irqreturn_t vepu_isr(int irq, void *dev_id);
2132 static void get_hw_info(struct vpu_subdev_data *data);
2133
2134 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2135 {
2136         struct device_node *dn = NULL;
2137         struct platform_device *pd = NULL;
2138         struct device *ret = NULL;
2139
2140         dn = of_find_compatible_node(NULL, NULL, compt);
2141         if (!dn) {
2142                 pr_err("can't find device node %s \r\n", compt);
2143                 return NULL;
2144         }
2145
2146         pd = of_find_device_by_node(dn);
2147         if (!pd) {
2148                 pr_err("can't find platform device in device node %s\n", compt);
2149                 return  NULL;
2150         }
2151         ret = &pd->dev;
2152
2153         return ret;
2154 }
2155
2156 #ifdef CONFIG_IOMMU_API
2157 static inline void platform_set_sysmmu(struct device *iommu,
2158                                        struct device *dev)
2159 {
2160         dev->archdata.iommu = iommu;
2161 }
2162 #else
2163 static inline void platform_set_sysmmu(struct device *iommu,
2164                                        struct device *dev)
2165 {
2166 }
2167 #endif
2168
2169 int vcodec_sysmmu_fault_hdl(struct device *dev,
2170                             enum rk_iommu_inttype itype,
2171                             unsigned long pgtable_base,
2172                             unsigned long fault_addr, unsigned int status)
2173 {
2174         struct platform_device *pdev;
2175         struct vpu_service_info *pservice;
2176         struct vpu_subdev_data *data;
2177
2178         vpu_debug_enter();
2179
2180         if (dev == NULL) {
2181                 pr_err("invalid NULL dev\n");
2182                 return 0;
2183         }
2184
2185         pdev = container_of(dev, struct platform_device, dev);
2186         if (pdev == NULL) {
2187                 pr_err("invalid NULL platform_device\n");
2188                 return 0;
2189         }
2190
2191         data = platform_get_drvdata(pdev);
2192         if (data == NULL) {
2193                 pr_err("invalid NULL vpu_subdev_data\n");
2194                 return 0;
2195         }
2196
2197         pservice = data->pservice;
2198         if (pservice == NULL) {
2199                 pr_err("invalid NULL vpu_service_info\n");
2200                 return 0;
2201         }
2202
2203         if (pservice->reg_codec) {
2204                 struct vpu_reg *reg = pservice->reg_codec;
2205                 struct vcodec_mem_region *mem, *n;
2206                 int i = 0;
2207
2208                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2209                 if (!list_empty(&reg->mem_region_list)) {
2210                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2211                                                  reg_lnk) {
2212                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2213                                        mem->reg_idx, i, mem->iova, mem->len);
2214                                 i++;
2215                         }
2216                 } else {
2217                         pr_err("no memory region mapped\n");
2218                 }
2219
2220                 if (reg->data) {
2221                         struct vpu_subdev_data *data = reg->data;
2222                         u32 *base = (u32 *)data->dec_dev.regs;
2223                         u32 len = data->hw_info->dec_reg_num;
2224
2225                         pr_err("current errror register set:\n");
2226
2227                         for (i = 0; i < len; i++)
2228                                 pr_err("reg[%02d] %08x\n",
2229                                        i, readl_relaxed(base + i));
2230                 }
2231
2232                 pr_alert("vcodec, page fault occur, reset hw\n");
2233
2234                 /* reg->reg[101] = 1; */
2235                 _vpu_reset(data);
2236         }
2237
2238         return 0;
2239 }
2240
2241 static int vcodec_subdev_probe(struct platform_device *pdev,
2242                                struct vpu_service_info *pservice)
2243 {
2244         uint8_t *regs = NULL;
2245         int32_t ret = 0;
2246         uint32_t ioaddr = 0;
2247         struct resource *res = NULL;
2248         struct vpu_hw_info *hw_info = NULL;
2249         struct device *dev = &pdev->dev;
2250         struct device_node *np = pdev->dev.of_node;
2251         struct vpu_subdev_data *data = NULL;
2252         struct platform_device *sub_dev = NULL;
2253         struct device_node *sub_np = NULL;
2254         const char *name  = np->name;
2255         char mmu_dev_dts_name[40];
2256
2257         dev_info(dev, "probe device");
2258
2259         data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2260         if (!data)
2261                 return -ENOMEM;
2262
2263         data->pservice = pservice;
2264         data->dev = dev;
2265         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2266
2267         if (pservice->reg_base == 0) {
2268                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2269                 data->regs = devm_ioremap_resource(dev, res);
2270                 if (IS_ERR(data->regs)) {
2271                         ret = PTR_ERR(data->regs);
2272                         goto err;
2273                 }
2274                 ioaddr = res->start;
2275         } else {
2276                 data->regs = pservice->reg_base;
2277                 ioaddr = pservice->ioaddr;
2278         }
2279
2280         sub_np = of_parse_phandle(np, "iommus", 0);
2281         if (sub_np) {
2282                 sub_dev = of_find_device_by_node(sub_np);
2283                 data->mmu_dev = &sub_dev->dev;
2284         }
2285
2286         /* Back to legacy iommu probe */
2287         if (!data->mmu_dev) {
2288                 switch (data->mode) {
2289                 case VCODEC_RUNNING_MODE_VPU:
2290                         sprintf(mmu_dev_dts_name,
2291                                 VPU_IOMMU_COMPATIBLE_NAME);
2292                         break;
2293                 case VCODEC_RUNNING_MODE_RKVDEC:
2294                         sprintf(mmu_dev_dts_name,
2295                                 VDEC_IOMMU_COMPATIBLE_NAME);
2296                         break;
2297                 case VCODEC_RUNNING_MODE_HEVC:
2298                 default:
2299                         sprintf(mmu_dev_dts_name,
2300                                 HEVC_IOMMU_COMPATIBLE_NAME);
2301                         break;
2302                 }
2303
2304                 data->mmu_dev =
2305                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2306                 if (data->mmu_dev)
2307                         platform_set_sysmmu(data->mmu_dev, dev);
2308
2309                 rockchip_iovmm_set_fault_handler
2310                         (dev, vcodec_sysmmu_fault_hdl);
2311         }
2312
2313         dev_info(dev, "vpu mmu dec %p\n", data->mmu_dev);
2314
2315         clear_bit(MMU_ACTIVATED, &data->state);
2316         vpu_service_power_on(data, pservice);
2317
2318         ret = vpu_service_check_hw(data);
2319         if (ret < 0) {
2320                 vpu_err("error: hw info check faild\n");
2321                 goto err;
2322         }
2323
2324         hw_info = data->hw_info;
2325         regs = (u8 *)data->regs;
2326
2327         if (hw_info->dec_reg_num) {
2328                 data->dec_dev.iosize = hw_info->dec_io_size;
2329                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2330         }
2331
2332         if (hw_info->enc_reg_num) {
2333                 data->enc_dev.iosize = hw_info->enc_io_size;
2334                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2335         }
2336
2337         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2338
2339         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2340         if (data->irq_enc > 0) {
2341                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2342                                                 vepu_irq, vepu_isr,
2343                                                 IRQF_SHARED, dev_name(dev),
2344                                                 (void *)data);
2345                 if (ret) {
2346                         dev_err(dev, "error: can't request vepu irq %d\n",
2347                                 data->irq_enc);
2348                         goto err;
2349                 }
2350         }
2351         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2352         if (data->irq_dec > 0) {
2353                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2354                                                 vdpu_irq, vdpu_isr,
2355                                                 IRQF_SHARED, dev_name(dev),
2356                                                 (void *)data);
2357                 if (ret) {
2358                         dev_err(dev, "error: can't request vdpu irq %d\n",
2359                                 data->irq_dec);
2360                         goto err;
2361                 }
2362         }
2363         atomic_set(&data->dec_dev.irq_count_codec, 0);
2364         atomic_set(&data->dec_dev.irq_count_pp, 0);
2365         atomic_set(&data->enc_dev.irq_count_codec, 0);
2366         atomic_set(&data->enc_dev.irq_count_pp, 0);
2367
2368         vcodec_enter_mode(data);
2369         of_property_read_u32(np, "allocator", (u32 *)&pservice->alloc_type);
2370         data->iommu_info = vcodec_iommu_info_create(dev, data->mmu_dev,
2371                                                     pservice->alloc_type);
2372         dev_info(dev, "allocator is %s\n", pservice->alloc_type == 1 ? "drm" :
2373                 (pservice->alloc_type == 2 ? "ion" : "null"));
2374         get_hw_info(data);
2375         pservice->auto_freq = true;
2376
2377         vcodec_exit_mode(data);
2378         /* create device node */
2379         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2380         if (ret) {
2381                 dev_err(dev, "alloc dev_t failed\n");
2382                 goto err;
2383         }
2384
2385         cdev_init(&data->cdev, &vpu_service_fops);
2386
2387         data->cdev.owner = THIS_MODULE;
2388         data->cdev.ops = &vpu_service_fops;
2389
2390         ret = cdev_add(&data->cdev, data->dev_t, 1);
2391
2392         if (ret) {
2393                 dev_err(dev, "add dev_t failed\n");
2394                 goto err;
2395         }
2396
2397         data->cls = class_create(THIS_MODULE, name);
2398
2399         if (IS_ERR(data->cls)) {
2400                 ret = PTR_ERR(data->cls);
2401                 dev_err(dev, "class_create err:%d\n", ret);
2402                 goto err;
2403         }
2404
2405         data->child_dev = device_create(data->cls, dev,
2406                 data->dev_t, "%s", name);
2407
2408         platform_set_drvdata(pdev, data);
2409
2410         INIT_LIST_HEAD(&data->lnk_service);
2411         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2412
2413         return 0;
2414 err:
2415         if (data->child_dev) {
2416                 device_destroy(data->cls, data->dev_t);
2417                 cdev_del(&data->cdev);
2418                 unregister_chrdev_region(data->dev_t, 1);
2419         }
2420
2421         if (data->cls)
2422                 class_destroy(data->cls);
2423         return -1;
2424 }
2425
2426 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2427 {
2428         struct vpu_service_info *pservice = data->pservice;
2429
2430         vcodec_iommu_info_destroy(data->iommu_info);
2431         data->iommu_info = NULL;
2432
2433         mutex_lock(&pservice->lock);
2434         cancel_delayed_work_sync(&pservice->power_off_work);
2435         vpu_service_power_off(pservice);
2436         mutex_unlock(&pservice->lock);
2437
2438         device_destroy(data->cls, data->dev_t);
2439         class_destroy(data->cls);
2440         cdev_del(&data->cdev);
2441         unregister_chrdev_region(data->dev_t, 1);
2442
2443 #ifdef CONFIG_DEBUG_FS
2444         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2445                 debugfs_remove_recursive(data->debugfs_dir);
2446 #endif
2447 }
2448
2449 static void vcodec_read_property(struct device_node *np,
2450                                  struct vpu_service_info *pservice)
2451 {
2452         pservice->mode_bit = 0;
2453         pservice->mode_ctrl = 0;
2454         pservice->subcnt = 0;
2455         pservice->grf_base = NULL;
2456
2457         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2458
2459         if (pservice->subcnt > 1) {
2460                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2461                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2462         }
2463 #ifdef CONFIG_MFD_SYSCON
2464         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2465         if (IS_ERR_OR_NULL(pservice->grf)) {
2466                 pservice->grf = NULL;
2467 #ifdef CONFIG_ARM
2468                 pservice->grf_base = RK_GRF_VIRT;
2469 #else
2470                 vpu_err("can't find vpu grf property\n");
2471                 return;
2472 #endif
2473         }
2474 #else
2475 #ifdef CONFIG_ARM
2476         pservice->grf_base = RK_GRF_VIRT;
2477 #else
2478         vpu_err("can't find vpu grf property\n");
2479         return;
2480 #endif
2481 #endif
2482
2483 #ifdef CONFIG_RESET_CONTROLLER
2484         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2485         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2486         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2487
2488         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2489                 dev_warn(pservice->dev, "No aclk reset resource define\n");
2490                 pservice->rst_a = NULL;
2491         }
2492
2493         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2494                 dev_warn(pservice->dev, "No hclk reset resource define\n");
2495                 pservice->rst_h = NULL;
2496         }
2497
2498         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2499                 dev_warn(pservice->dev, "No core reset resource define\n");
2500                 pservice->rst_v = NULL;
2501         }
2502 #endif
2503
2504         of_property_read_string(np, "name", (const char **)&pservice->name);
2505 }
2506
2507 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2508 {
2509         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2510         pservice->curr_mode = -1;
2511
2512         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2513         INIT_LIST_HEAD(&pservice->waiting);
2514         INIT_LIST_HEAD(&pservice->running);
2515         mutex_init(&pservice->lock);
2516         mutex_init(&pservice->shutdown_lock);
2517         atomic_set(&pservice->service_on, 1);
2518
2519         INIT_LIST_HEAD(&pservice->done);
2520         INIT_LIST_HEAD(&pservice->session);
2521         INIT_LIST_HEAD(&pservice->subdev_list);
2522
2523         pservice->reg_pproc     = NULL;
2524         atomic_set(&pservice->total_running, 0);
2525         atomic_set(&pservice->enabled,       0);
2526         atomic_set(&pservice->power_on_cnt,  0);
2527         atomic_set(&pservice->power_off_cnt, 0);
2528         atomic_set(&pservice->reset_request, 0);
2529
2530         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2531         pservice->last.tv64 = 0;
2532
2533         pservice->alloc_type = 0;
2534 }
2535
2536 static int vcodec_probe(struct platform_device *pdev)
2537 {
2538         int i;
2539         int ret = 0;
2540         struct resource *res = NULL;
2541         struct device *dev = &pdev->dev;
2542         struct device_node *np = pdev->dev.of_node;
2543         struct vpu_service_info *pservice = NULL;
2544         struct vcodec_device_info *driver_data;
2545
2546         pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info),
2547                                 GFP_KERNEL);
2548         if (!pservice)
2549                 return -ENOMEM;
2550         pservice->dev = dev;
2551
2552         driver_data = vcodec_get_drv_data(pdev);
2553         if (!driver_data)
2554                 return -EINVAL;
2555
2556         vcodec_read_property(np, pservice);
2557         vcodec_init_drvdata(pservice);
2558
2559         /* Underscore for label, hyphens for name */
2560         switch (driver_data->device_type) {
2561         case VCODEC_DEVICE_TYPE_VPUX:
2562                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2563                 break;
2564         case VCODEC_DEVICE_TYPE_VPUC:
2565                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2566                 break;
2567         case VCODEC_DEVICE_TYPE_HEVC:
2568                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2569                 break;
2570         case VCODEC_DEVICE_TYPE_RKVD:
2571                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2572                 break;
2573         default:
2574                 dev_err(dev, "unsupported device type\n");
2575                 return -ENODEV;
2576         }
2577
2578         if (0 > vpu_get_clk(pservice))
2579                 goto err;
2580
2581         if (of_property_read_bool(np, "reg")) {
2582                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2583
2584                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2585                 if (IS_ERR(pservice->reg_base)) {
2586                         vpu_err("ioremap registers base failed\n");
2587                         ret = PTR_ERR(pservice->reg_base);
2588                         goto err;
2589                 }
2590                 pservice->ioaddr = res->start;
2591         } else {
2592                 pservice->reg_base = 0;
2593         }
2594
2595         pm_runtime_enable(dev);
2596
2597         if (of_property_read_bool(np, "subcnt")) {
2598                 for (i = 0; i < pservice->subcnt; i++) {
2599                         struct device_node *sub_np;
2600                         struct platform_device *sub_pdev;
2601
2602                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2603                         sub_pdev = of_find_device_by_node(sub_np);
2604
2605                         vcodec_subdev_probe(sub_pdev, pservice);
2606                 }
2607         } else {
2608                 vcodec_subdev_probe(pdev, pservice);
2609         }
2610
2611         vpu_service_power_off(pservice);
2612
2613         dev_info(dev, "init success\n");
2614
2615         return 0;
2616
2617 err:
2618         dev_info(dev, "init failed\n");
2619         vpu_service_power_off(pservice);
2620         wake_lock_destroy(&pservice->wake_lock);
2621
2622         return ret;
2623 }
2624
2625 static int vcodec_remove(struct platform_device *pdev)
2626 {
2627         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2628
2629         vcodec_subdev_remove(data);
2630
2631         pm_runtime_disable(data->pservice->dev);
2632
2633         return 0;
2634 }
2635
2636 static void vcodec_shutdown(struct platform_device *pdev)
2637 {
2638         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2639         struct vpu_service_info *pservice = data->pservice;
2640         int val;
2641         int ret;
2642
2643         dev_info(&pdev->dev, "vcodec shutdown");
2644
2645         mutex_lock(&pservice->shutdown_lock);
2646         atomic_set(&pservice->service_on, 0);
2647         mutex_unlock(&pservice->shutdown_lock);
2648
2649         ret = readx_poll_timeout(atomic_read,
2650                                  &pservice->total_running,
2651                                  val, val == 0, 20000, 200000);
2652         if (ret == -ETIMEDOUT)
2653                 dev_err(&pdev->dev, "wait total running time out\n");
2654
2655         vcodec_exit_mode(data);
2656
2657         vpu_service_power_on(data, pservice);
2658         vpu_service_clear(data);
2659         vcodec_subdev_remove(data);
2660
2661         pm_runtime_disable(&pdev->dev);
2662 }
2663
2664 static const struct of_device_id vcodec_service_dt_ids[] = {
2665         {
2666                 .compatible = "rockchip,vpu_service",
2667                 .data = &vpu_device_info,
2668         },
2669         {
2670                 .compatible = "rockchip,hevc_service",
2671                 .data = &hevc_device_info,
2672         },
2673         {
2674                 .compatible = "rockchip,vpu_combo",
2675                 .data = &vpu_combo_device_info,
2676         },
2677         {
2678                 .compatible = "rockchip,rkvdec",
2679                 .data = &rkvd_device_info,
2680         },
2681         {},
2682 };
2683
2684 MODULE_DEVICE_TABLE(of, vcodec_service_dt_ids);
2685
2686 static void *vcodec_get_drv_data(struct platform_device *pdev)
2687 {
2688         struct vcodec_device_info *driver_data = NULL;
2689         const struct of_device_id *match;
2690
2691         match = of_match_node(vcodec_service_dt_ids, pdev->dev.of_node);
2692         if (match)
2693                 driver_data = (struct vcodec_device_info *)match->data;
2694
2695         return driver_data;
2696 }
2697
2698 static struct platform_driver vcodec_driver = {
2699         .probe = vcodec_probe,
2700         .remove = vcodec_remove,
2701         .shutdown = vcodec_shutdown,
2702         .driver = {
2703                 .name = "rk-vcodec",
2704                 .owner = THIS_MODULE,
2705                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2706         },
2707 };
2708
2709 static void get_hw_info(struct vpu_subdev_data *data)
2710 {
2711         struct vpu_service_info *pservice = data->pservice;
2712         struct vpu_dec_config *dec = &pservice->dec_config;
2713         struct vpu_enc_config *enc = &pservice->enc_config;
2714
2715         if (of_machine_is_compatible("rockchip,rk2928") ||
2716                         of_machine_is_compatible("rockchip,rk3036") ||
2717                         of_machine_is_compatible("rockchip,rk3066") ||
2718                         of_machine_is_compatible("rockchip,rk3126") ||
2719                         of_machine_is_compatible("rockchip,rk3188"))
2720                 dec->max_dec_pic_width = 1920;
2721         else
2722                 dec->max_dec_pic_width = 4096;
2723
2724         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2725                 dec->h264_support = 3;
2726                 dec->jpeg_support = 1;
2727                 dec->mpeg4_support = 2;
2728                 dec->vc1_support = 3;
2729                 dec->mpeg2_support = 1;
2730                 dec->pp_support = 1;
2731                 dec->sorenson_support = 1;
2732                 dec->ref_buf_support = 3;
2733                 dec->vp6_support = 1;
2734                 dec->vp7_support = 1;
2735                 dec->vp8_support = 1;
2736                 dec->avs_support = 1;
2737                 dec->jpeg_ext_support = 0;
2738                 dec->custom_mpeg4_support = 1;
2739                 dec->reserve = 0;
2740                 dec->mvc_support = 1;
2741
2742                 if (!of_machine_is_compatible("rockchip,rk3036")) {
2743                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2744
2745                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2746                         enc->h264_enabled = 1;
2747                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2748                         enc->jpeg_enabled = 1;
2749                         enc->vs_enabled = (config_reg >> 24) & 1;
2750                         enc->rgb_enabled = (config_reg >> 28) & 1;
2751                         enc->reg_size = data->reg_size;
2752                         enc->reserv[0] = 0;
2753                         enc->reserv[1] = 0;
2754                 }
2755
2756                 pservice->auto_freq = true;
2757                 vpu_debug(DEBUG_EXTRA_INFO, "vpu_service set to auto frequency mode\n");
2758                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2759
2760                 pservice->bug_dec_addr = of_machine_is_compatible
2761                         ("rockchip,rk30xx");
2762         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2763                 pservice->auto_freq = true;
2764                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2765         } else {
2766                 /* disable frequency switch in hevc.*/
2767                 pservice->auto_freq = false;
2768         }
2769 }
2770
2771 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2772 {
2773         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2774                   task->name, irq_status, task->error_mask);
2775
2776         return (task->error_mask & irq_status) ? true : false;
2777 }
2778
2779 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2780 {
2781         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2782         struct vpu_service_info *pservice = data->pservice;
2783         struct vpu_task_info *task = NULL;
2784         struct vpu_device *dev = &data->dec_dev;
2785         u32 hw_id = data->hw_info->hw_id;
2786         u32 raw_status;
2787         u32 dec_status;
2788
2789         task = &data->task_info[TASK_DEC];
2790
2791         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2792         dec_status = raw_status;
2793
2794         vpu_debug(DEBUG_TASK_INFO, "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2795                   task->reg_irq, dec_status,
2796                   task->irq_mask, task->ready_mask, task->error_mask);
2797
2798         if (dec_status & task->irq_mask) {
2799                 time_record(task, 1);
2800                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2801                           dec_status);
2802                 if ((dec_status & 0x40001) == 0x40001) {
2803                         do {
2804                                 dec_status =
2805                                         readl_relaxed(dev->regs +
2806                                                 task->reg_irq);
2807                         } while ((dec_status & 0x40001) == 0x40001);
2808                 }
2809
2810                 if (check_irq_err(task, dec_status))
2811                         atomic_add(1, &pservice->reset_request);
2812
2813                 writel_relaxed(0, dev->regs + task->reg_irq);
2814
2815                 /* set clock gating to save power */
2816                 writel(task->gating_mask, dev->regs + task->reg_en);
2817
2818                 atomic_add(1, &dev->irq_count_codec);
2819                 time_diff(task);
2820         }
2821
2822         task = &data->task_info[TASK_PP];
2823         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2824                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2825
2826                 if (pp_status & task->irq_mask) {
2827                         time_record(task, 1);
2828                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2829                                   pp_status);
2830
2831                         if (check_irq_err(task, dec_status))
2832                                 atomic_add(1, &pservice->reset_request);
2833
2834                         /* clear pp IRQ */
2835                         writel_relaxed(pp_status & (~task->reg_irq),
2836                                        dev->regs + task->irq_mask);
2837                         atomic_add(1, &dev->irq_count_pp);
2838                         time_diff(task);
2839                 }
2840         }
2841
2842         pservice->irq_status = raw_status;
2843
2844         if (atomic_read(&dev->irq_count_pp) ||
2845             atomic_read(&dev->irq_count_codec))
2846                 return IRQ_WAKE_THREAD;
2847         else
2848                 return IRQ_NONE;
2849 }
2850
2851 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2852 {
2853         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2854         struct vpu_service_info *pservice = data->pservice;
2855         struct vpu_device *dev = &data->dec_dev;
2856
2857         mutex_lock(&pservice->lock);
2858         if (atomic_read(&dev->irq_count_codec)) {
2859                 atomic_sub(1, &dev->irq_count_codec);
2860                 if (pservice->reg_codec == NULL) {
2861                         vpu_err("error: dec isr with no task waiting\n");
2862                 } else {
2863                         reg_from_run_to_done(data, pservice->reg_codec);
2864                         /* avoid vpu timeout and can't recover problem */
2865                         if (data->mode == VCODEC_RUNNING_MODE_VPU)
2866                                 VDPU_SOFT_RESET(data->regs);
2867                 }
2868         }
2869
2870         if (atomic_read(&dev->irq_count_pp)) {
2871                 atomic_sub(1, &dev->irq_count_pp);
2872                 if (pservice->reg_pproc == NULL)
2873                         vpu_err("error: pp isr with no task waiting\n");
2874                 else
2875                         reg_from_run_to_done(data, pservice->reg_pproc);
2876         }
2877         try_set_reg(data);
2878         mutex_unlock(&pservice->lock);
2879         return IRQ_HANDLED;
2880 }
2881
2882 static irqreturn_t vepu_irq(int irq, void *dev_id)
2883 {
2884         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2885         struct vpu_service_info *pservice = data->pservice;
2886         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2887         struct vpu_device *dev = &data->enc_dev;
2888         u32 irq_status;
2889
2890         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2891
2892         vpu_debug(DEBUG_TASK_INFO, "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2893                   task->reg_irq, irq_status,
2894                   task->irq_mask, task->ready_mask, task->error_mask);
2895
2896         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2897
2898         if (likely(irq_status & task->irq_mask)) {
2899                 time_record(task, 1);
2900
2901                 if (check_irq_err(task, irq_status))
2902                         atomic_add(1, &pservice->reset_request);
2903
2904                 /* clear enc IRQ */
2905                 writel_relaxed(irq_status & (~task->irq_mask),
2906                                dev->regs + task->reg_irq);
2907
2908                 atomic_add(1, &dev->irq_count_codec);
2909                 time_diff(task);
2910         }
2911
2912         pservice->irq_status = irq_status;
2913
2914         if (atomic_read(&dev->irq_count_codec))
2915                 return IRQ_WAKE_THREAD;
2916         else
2917                 return IRQ_NONE;
2918 }
2919
2920 static irqreturn_t vepu_isr(int irq, void *dev_id)
2921 {
2922         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2923         struct vpu_service_info *pservice = data->pservice;
2924         struct vpu_device *dev = &data->enc_dev;
2925
2926         mutex_lock(&pservice->lock);
2927         if (atomic_read(&dev->irq_count_codec)) {
2928                 atomic_sub(1, &dev->irq_count_codec);
2929                 if (NULL == pservice->reg_codec)
2930                         vpu_err("error: enc isr with no task waiting\n");
2931                 else
2932                         reg_from_run_to_done(data, pservice->reg_codec);
2933         }
2934         try_set_reg(data);
2935         mutex_unlock(&pservice->lock);
2936         return IRQ_HANDLED;
2937 }
2938
2939 module_platform_driver(vcodec_driver);
2940 MODULE_LICENSE("GPL v2");