drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/clk.h>
  20 #include <linux/compat.h>
  21 #include <linux/delay.h>
  22 #include <linux/init.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/module.h>
  25 #include <linux/fs.h>
  26 #include <linux/mm.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/reset.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31 #include <linux/wakelock.h>
  32 #include <linux/cdev.h>
  33 #include <linux/of.h>
  34 #include <linux/of_platform.h>
  35 #include <linux/of_irq.h>
  36 #include <linux/regmap.h>
  37 #include <linux/mfd/syscon.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/debugfs.h>
  40 #include <linux/pm_runtime.h>
  41
  42 #include <linux/rockchip/cpu.h>
  43 #include <linux/rockchip/cru.h>
  44 #include <linux/rockchip/pmu.h>
  45 #include <linux/rockchip/grf.h>
  46
  47 #if defined(CONFIG_ION_ROCKCHIP)
  48 #include <linux/rockchip_ion.h>
  49 #endif
  50
  51 #include <linux/rockchip-iovmm.h>
  52 #include <linux/dma-buf.h>
  53
  54 #include "vcodec_hw_info.h"
  55 #include "vcodec_hw_vpu.h"
  56 #include "vcodec_hw_rkv.h"
  57 #include "vcodec_hw_vpu2.h"
  58
  59 #include "vcodec_service.h"
  60
  61 /*
  62  * debug flag usage:
  63  * +------+-------------------+
  64  * | 8bit |      24bit        |
  65  * +------+-------------------+
  66  *  0~23 bit is for different information type
  67  * 24~31 bit is for information print format
  68  */
  69
  70 #define DEBUG_POWER                             0x00000001
  71 #define DEBUG_CLOCK                             0x00000002
  72 #define DEBUG_IRQ_STATUS                        0x00000004
  73 #define DEBUG_IOMMU                             0x00000008
  74 #define DEBUG_IOCTL                             0x00000010
  75 #define DEBUG_FUNCTION                          0x00000020
  76 #define DEBUG_REGISTER                          0x00000040
  77 #define DEBUG_EXTRA_INFO                        0x00000080
  78 #define DEBUG_TIMING                            0x00000100
  79 #define DEBUG_TASK_INFO                         0x00000200
  80
  81 #define DEBUG_SET_REG                           0x00001000
  82 #define DEBUG_GET_REG                           0x00002000
  83 #define DEBUG_PPS_FILL                          0x00004000
  84 #define DEBUG_IRQ_CHECK                         0x00008000
  85 #define DEBUG_CACHE_32B                         0x00010000
  86
  87 #define PRINT_FUNCTION                          0x80000000
  88 #define PRINT_LINE                              0x40000000
  89
  90 static int debug;
  91 module_param(debug, int, S_IRUGO | S_IWUSR);
  92 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  93
  94 #define VCODEC_CLOCK_ENABLE     1
  95
  96 /*
  97  * hardware information organization
  98  *
  99  * In order to support multiple hardware with different version the hardware
 100  * information is organized as follow:
 101  *
 102  * 1. First, index hardware by register size / position.
 103  *    These information is fix for each hardware and do not relate to runtime
 104  *    work flow. It only related to resource allocation.
 105  *    Descriptor: struct vpu_hw_info
 106  *
 107  * 2. Then, index hardware by runtime configuration
 108  *    These information is related to runtime setting behave including enable
 109  *    register, irq register and other key control flag
 110  *    Descriptor: struct vpu_task_info
 111  *
 112  * 3. Final, on iommu case the fd translation is required
 113  *    Descriptor: struct vpu_trans_info
 114  */
 115
 116 enum VPU_FREQ {
 117         VPU_FREQ_200M,
 118         VPU_FREQ_266M,
 119         VPU_FREQ_300M,
 120         VPU_FREQ_400M,
 121         VPU_FREQ_500M,
 122         VPU_FREQ_600M,
 123         VPU_FREQ_DEFAULT,
 124         VPU_FREQ_BUT,
 125 };
 126
 127 struct extra_info_elem {
 128         u32 index;
 129         u32 offset;
 130 };
 131
 132 #define EXTRA_INFO_MAGIC        0x4C4A46
 133
 134 struct extra_info_for_iommu {
 135         u32 magic;
 136         u32 cnt;
 137         struct extra_info_elem elem[20];
 138 };
 139
 140 #define MHZ                                     (1000*1000)
 141 #define SIZE_REG(reg)                           ((reg)*4)
 142
 143 static struct vcodec_info vcodec_info_set[] = {
 144         [0] = {
 145                 .hw_id          = VPU_ID_8270,
 146                 .hw_info        = &hw_vpu_8270,
 147                 .task_info      = task_vpu,
 148                 .trans_info     = trans_vpu,
 149         },
 150         [1] = {
 151                 .hw_id          = VPU_ID_4831,
 152                 .hw_info        = &hw_vpu_4831,
 153                 .task_info      = task_vpu,
 154                 .trans_info     = trans_vpu,
 155         },
 156         [2] = {
 157                 .hw_id          = VPU_DEC_ID_9190,
 158                 .hw_info        = &hw_vpu_9190,
 159                 .task_info      = task_vpu,
 160                 .trans_info     = trans_vpu,
 161         },
 162         [3] = {
 163                 .hw_id          = HEVC_ID,
 164                 .hw_info        = &hw_rkhevc,
 165                 .task_info      = task_rkv,
 166                 .trans_info     = trans_rkv,
 167         },
 168         [4] = {
 169                 .hw_id          = RKV_DEC_ID,
 170                 .hw_info        = &hw_rkvdec,
 171                 .task_info      = task_rkv,
 172                 .trans_info     = trans_rkv,
 173         },
 174         [5] = {
 175                 .hw_id          = VPU2_ID,
 176                 .hw_info        = &hw_vpu2,
 177                 .task_info      = task_vpu2,
 178                 .trans_info     = trans_vpu2,
 179         },
 180 };
 181
 182 #define DEBUG
 183 #ifdef DEBUG
 184 #define vpu_debug_func(type, fmt, args...)                      \
 185         do {                                                    \
 186                 if (unlikely(debug & type)) {                   \
 187                         pr_info("%s:%d: " fmt,                  \
 188                                  __func__, __LINE__, ##args);   \
 189                 }                                               \
 190         } while (0)
 191 #define vpu_debug(type, fmt, args...)                           \
 192         do {                                                    \
 193                 if (unlikely(debug & type)) {                   \
 194                         pr_info(fmt, ##args);                   \
 195                 }                                               \
 196         } while (0)
 197 #else
 198 #define vpu_debug_func(level, fmt, args...)
 199 #define vpu_debug(level, fmt, args...)
 200 #endif
 201
 202 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 203 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 204
 205 #define vpu_err(fmt, args...)                           \
 206                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 207
 208 enum VPU_DEC_FMT {
 209         VPU_DEC_FMT_H264,
 210         VPU_DEC_FMT_MPEG4,
 211         VPU_DEC_FMT_H263,
 212         VPU_DEC_FMT_JPEG,
 213         VPU_DEC_FMT_VC1,
 214         VPU_DEC_FMT_MPEG2,
 215         VPU_DEC_FMT_MPEG1,
 216         VPU_DEC_FMT_VP6,
 217         VPU_DEC_FMT_RESERV0,
 218         VPU_DEC_FMT_VP7,
 219         VPU_DEC_FMT_VP8,
 220         VPU_DEC_FMT_AVS,
 221         VPU_DEC_FMT_RES
 222 };
 223
 224 /**
 225  * struct for process session which connect to vpu
 226  *
 227  * @author ChenHengming (2011-5-3)
 228  */
 229 struct vpu_session {
 230         enum VPU_CLIENT_TYPE type;
 231         /* a linked list of data so we can access them for debugging */
 232         struct list_head list_session;
 233         /* a linked list of register data waiting for process */
 234         struct list_head waiting;
 235         /* a linked list of register data in processing */
 236         struct list_head running;
 237         /* a linked list of register data processed */
 238         struct list_head done;
 239         wait_queue_head_t wait;
 240         pid_t pid;
 241         atomic_t task_running;
 242 };
 243
 244 /**
 245  * struct for process register set
 246  *
 247  * @author ChenHengming (2011-5-4)
 248  */
 249 struct vpu_reg {
 250         enum VPU_CLIENT_TYPE type;
 251         enum VPU_FREQ freq;
 252         struct vpu_session *session;
 253         struct vpu_subdev_data *data;
 254         struct vpu_task_info *task;
 255         const struct vpu_trans_info *trans;
 256
 257         /* link to vpu service session */
 258         struct list_head session_link;
 259         /* link to register set list */
 260         struct list_head status_link;
 261
 262         unsigned long size;
 263         struct list_head mem_region_list;
 264         u32 dec_base;
 265         u32 *reg;
 266 };
 267
 268 struct vpu_device {
 269         atomic_t irq_count_codec;
 270         atomic_t irq_count_pp;
 271         unsigned int iosize;
 272         u32 *regs;
 273 };
 274
 275 enum vcodec_device_id {
 276         VCODEC_DEVICE_ID_VPU,
 277         VCODEC_DEVICE_ID_HEVC,
 278         VCODEC_DEVICE_ID_COMBO,
 279         VCODEC_DEVICE_ID_RKVDEC,
 280         VCODEC_DEVICE_ID_BUTT
 281 };
 282
 283 enum VCODEC_RUNNING_MODE {
 284         VCODEC_RUNNING_MODE_NONE = -1,
 285         VCODEC_RUNNING_MODE_VPU,
 286         VCODEC_RUNNING_MODE_HEVC,
 287         VCODEC_RUNNING_MODE_RKVDEC
 288 };
 289
 290 struct vcodec_mem_region {
 291         struct list_head srv_lnk;
 292         struct list_head reg_lnk;
 293         struct list_head session_lnk;
 294         unsigned long iova;     /* virtual address for iommu */
 295         unsigned long len;
 296         u32 reg_idx;
 297         struct ion_handle *hdl;
 298 };
 299
 300 enum vpu_ctx_state {
 301         MMU_ACTIVATED   = BIT(0)
 302 };
 303
 304 struct vpu_subdev_data {
 305         struct cdev cdev;
 306         dev_t dev_t;
 307         struct class *cls;
 308         struct device *child_dev;
 309
 310         int irq_enc;
 311         int irq_dec;
 312         struct vpu_service_info *pservice;
 313
 314         u32 *regs;
 315         enum VCODEC_RUNNING_MODE mode;
 316         struct list_head lnk_service;
 317
 318         struct device *dev;
 319
 320         struct vpu_device enc_dev;
 321         struct vpu_device dec_dev;
 322
 323         enum VPU_HW_ID hw_id;
 324         struct vpu_hw_info *hw_info;
 325         struct vpu_task_info *task_info;
 326         const struct vpu_trans_info *trans_info;
 327
 328         u32 reg_size;
 329         unsigned long state;
 330
 331 #ifdef CONFIG_DEBUG_FS
 332         struct dentry *debugfs_dir;
 333         struct dentry *debugfs_file_regs;
 334 #endif
 335
 336         struct device *mmu_dev;
 337 };
 338
 339 struct vpu_service_info {
 340         struct wake_lock wake_lock;
 341         struct delayed_work power_off_work;
 342         ktime_t last; /* record previous power-on time */
 343         /* vpu service structure global lock */
 344         struct mutex lock;
 345         /* link to link_reg in struct vpu_reg */
 346         struct list_head waiting;
 347         /* link to link_reg in struct vpu_reg */
 348         struct list_head running;
 349         /* link to link_reg in struct vpu_reg */
 350         struct list_head done;
 351         /* link to list_session in struct vpu_session */
 352         struct list_head session;
 353         atomic_t total_running;
 354         atomic_t enabled;
 355         atomic_t power_on_cnt;
 356         atomic_t power_off_cnt;
 357         atomic_t service_on;
 358         struct mutex shutdown_lock;
 359         struct vpu_reg *reg_codec;
 360         struct vpu_reg *reg_pproc;
 361         struct vpu_reg *reg_resev;
 362         struct vpu_dec_config dec_config;
 363         struct vpu_enc_config enc_config;
 364
 365         bool auto_freq;
 366         bool bug_dec_addr;
 367         atomic_t freq_status;
 368
 369         struct clk *aclk_vcodec;
 370         struct clk *hclk_vcodec;
 371         struct clk *clk_core;
 372         struct clk *clk_cabac;
 373         struct clk *pd_video;
 374
 375 #ifdef CONFIG_RESET_CONTROLLER
 376         struct reset_control *rst_a;
 377         struct reset_control *rst_h;
 378         struct reset_control *rst_v;
 379 #endif
 380         struct device *dev;
 381
 382         u32 irq_status;
 383         atomic_t reset_request;
 384         struct ion_client *ion_client;
 385         struct list_head mem_region_list;
 386
 387         enum vcodec_device_id dev_id;
 388
 389         enum VCODEC_RUNNING_MODE curr_mode;
 390         u32 prev_mode;
 391
 392         struct delayed_work simulate_work;
 393
 394         u32 mode_bit;
 395         u32 mode_ctrl;
 396         u32 *reg_base;
 397         u32 ioaddr;
 398         struct regmap *grf;
 399         u32 *grf_base;
 400
 401         char *name;
 402
 403         u32 subcnt;
 404         struct list_head subdev_list;
 405 };
 406
 407 struct vpu_request {
 408         u32 *req;
 409         u32 size;
 410 };
 411
 412 #ifdef CONFIG_COMPAT
 413 struct compat_vpu_request {
 414         compat_uptr_t req;
 415         u32 size;
 416 };
 417 #endif
 418
 419 /* debugfs root directory for all device (vpu, hevc).*/
 420 static struct dentry *parent;
 421
 422 #ifdef CONFIG_DEBUG_FS
 423 static int vcodec_debugfs_init(void);
 424 static void vcodec_debugfs_exit(void);
 425 static struct dentry *vcodec_debugfs_create_device_dir(
 426                 char *dirname, struct dentry *parent);
 427 static int debug_vcodec_open(struct inode *inode, struct file *file);
 428
 429 static const struct file_operations debug_vcodec_fops = {
 430         .open = debug_vcodec_open,
 431         .read = seq_read,
 432         .llseek = seq_lseek,
 433         .release = single_release,
 434 };
 435 #endif
 436
 437 #define VDPU_SOFT_RESET_REG     101
 438 #define VDPU_CLEAN_CACHE_REG    516
 439 #define VEPU_CLEAN_CACHE_REG    772
 440 #define HEVC_CLEAN_CACHE_REG    260
 441
 442 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 443
 444 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 445 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 446 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 447 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 448
 449 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 450 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 451
 452 static void time_record(struct vpu_task_info *task, int is_end)
 453 {
 454         if (unlikely(debug & DEBUG_TIMING) && task)
 455                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 456 }
 457
 458 static void time_diff(struct vpu_task_info *task)
 459 {
 460         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 461                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 462                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 463 }
 464
 465 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 466 {
 467         int bits;
 468         u32 raw = 0;
 469         struct vpu_service_info *pservice = data->pservice;
 470         struct vpu_subdev_data *subdata, *n;
 471
 472         if (pservice->subcnt < 2) {
 473                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 474                         set_bit(MMU_ACTIVATED, &data->state);
 475                         if (atomic_read(&pservice->enabled))
 476                                 rockchip_iovmm_activate(data->dev);
 477                         else
 478                                 BUG_ON(!atomic_read(&pservice->enabled));
 479                 }
 480                 return;
 481         }
 482
 483         if (pservice->curr_mode == data->mode)
 484                 return;
 485
 486         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 487         list_for_each_entry_safe(subdata, n,
 488                                  &pservice->subdev_list, lnk_service) {
 489                 if (data != subdata && subdata->mmu_dev &&
 490                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 491                         clear_bit(MMU_ACTIVATED, &subdata->state);
 492                         rockchip_iovmm_deactivate(subdata->dev);
 493                 }
 494         }
 495         bits = 1 << pservice->mode_bit;
 496 #ifdef CONFIG_MFD_SYSCON
 497         if (pservice->grf) {
 498                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 499
 500                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 501                         regmap_write(pservice->grf, pservice->mode_ctrl,
 502                                      raw | bits | (bits << 16));
 503                 else
 504                         regmap_write(pservice->grf, pservice->mode_ctrl,
 505                                      (raw & (~bits)) | (bits << 16));
 506         } else if (pservice->grf_base) {
 507                 u32 *grf_base = pservice->grf_base;
 508
 509                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 510                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 511                         writel_relaxed(raw | bits | (bits << 16),
 512                                        grf_base + pservice->mode_ctrl / 4);
 513                 else
 514                         writel_relaxed((raw & (~bits)) | (bits << 16),
 515                                        grf_base + pservice->mode_ctrl / 4);
 516         } else {
 517                 vpu_err("no grf resource define, switch decoder failed\n");
 518                 return;
 519         }
 520 #else
 521         if (pservice->grf_base) {
 522                 u32 *grf_base = pservice->grf_base;
 523
 524                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 525                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 526                         writel_relaxed(raw | bits | (bits << 16),
 527                                        grf_base + pservice->mode_ctrl / 4);
 528                 else
 529                         writel_relaxed((raw & (~bits)) | (bits << 16),
 530                                        grf_base + pservice->mode_ctrl / 4);
 531         } else {
 532                 vpu_err("no grf resource define, switch decoder failed\n");
 533                 return;
 534         }
 535 #endif
 536         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 537                 set_bit(MMU_ACTIVATED, &data->state);
 538                 if (atomic_read(&pservice->enabled))
 539                         rockchip_iovmm_activate(data->dev);
 540                 else
 541                         BUG_ON(!atomic_read(&pservice->enabled));
 542         }
 543
 544         pservice->prev_mode = pservice->curr_mode;
 545         pservice->curr_mode = data->mode;
 546 }
 547
 548 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 549 {
 550         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 551                 clear_bit(MMU_ACTIVATED, &data->state);
 552                 rockchip_iovmm_deactivate(data->dev);
 553         }
 554         /*
 555          * In case of VPU Combo, it require HW switch its running mode
 556          * before the other HW component start work. set current HW running
 557          * mode to none, can ensure HW switch to its reqired mode properly.
 558          */
 559         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 560 }
 561
 562 static int vpu_get_clk(struct vpu_service_info *pservice)
 563 {
 564 #if VCODEC_CLOCK_ENABLE
 565         struct device *dev = pservice->dev;
 566
 567         switch (pservice->dev_id) {
 568         case VCODEC_DEVICE_ID_HEVC:
 569                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 570                 if (IS_ERR(pservice->pd_video)) {
 571                         dev_err(dev, "failed on clk_get pd_hevc\n");
 572                         return -1;
 573                 }
 574         case VCODEC_DEVICE_ID_COMBO:
 575         case VCODEC_DEVICE_ID_RKVDEC:
 576                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 577                 if (IS_ERR(pservice->clk_cabac)) {
 578                         dev_err(dev, "failed on clk_get clk_cabac\n");
 579                         pservice->clk_cabac = NULL;
 580                 }
 581                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 582                 if (IS_ERR(pservice->clk_core)) {
 583                         dev_err(dev, "failed on clk_get clk_core\n");
 584                         return -1;
 585                 }
 586         case VCODEC_DEVICE_ID_VPU:
 587                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 588                 if (IS_ERR(pservice->aclk_vcodec)) {
 589                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 590                         return -1;
 591                 }
 592
 593                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 594                 if (IS_ERR(pservice->hclk_vcodec)) {
 595                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 596                         return -1;
 597                 }
 598                 if (pservice->pd_video == NULL) {
 599                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 600                         if (IS_ERR(pservice->pd_video)) {
 601                                 pservice->pd_video = NULL;
 602                                 dev_info(dev, "do not have pd_video\n");
 603                         }
 604                 }
 605                 break;
 606         default:
 607                 break;
 608         }
 609
 610         return 0;
 611 #else
 612         return 0;
 613 #endif
 614 }
 615
 616 static void vpu_put_clk(struct vpu_service_info *pservice)
 617 {
 618 #if VCODEC_CLOCK_ENABLE
 619         if (pservice->pd_video)
 620                 devm_clk_put(pservice->dev, pservice->pd_video);
 621         if (pservice->aclk_vcodec)
 622                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 623         if (pservice->hclk_vcodec)
 624                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 625         if (pservice->clk_core)
 626                 devm_clk_put(pservice->dev, pservice->clk_core);
 627         if (pservice->clk_cabac)
 628                 devm_clk_put(pservice->dev, pservice->clk_cabac);
 629 #endif
 630 }
 631
 632 static void vpu_reset(struct vpu_subdev_data *data)
 633 {
 634         struct vpu_service_info *pservice = data->pservice;
 635         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 636
 637         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 638                 type = IDLE_REQ_HEVC;
 639
 640         pr_info("%s: resetting...", dev_name(pservice->dev));
 641
 642 #if defined(CONFIG_ARCH_RK29)
 643         clk_disable(aclk_ddr_vepu);
 644         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 645         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 646         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 647         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 648         mdelay(10);
 649         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 650         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 651         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 652         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 653         clk_enable(aclk_ddr_vepu);
 654 #elif defined(CONFIG_ARCH_RK30)
 655         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 656         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 657         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 658         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 659         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 660         mdelay(1);
 661         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 662         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 663         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 664         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 665         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 666 #else
 667 #endif
 668         WARN_ON(pservice->reg_codec != NULL);
 669         WARN_ON(pservice->reg_pproc != NULL);
 670         WARN_ON(pservice->reg_resev != NULL);
 671         pservice->reg_codec = NULL;
 672         pservice->reg_pproc = NULL;
 673         pservice->reg_resev = NULL;
 674
 675         pr_info("for 3288/3368...");
 676 #ifdef CONFIG_RESET_CONTROLLER
 677         if (pservice->rst_a && pservice->rst_h) {
 678                 pr_info("reset in\n");
 679                 if (pservice->rst_v)
 680                         reset_control_assert(pservice->rst_v);
 681                 reset_control_assert(pservice->rst_a);
 682                 reset_control_assert(pservice->rst_h);
 683                 udelay(5);
 684                 reset_control_deassert(pservice->rst_h);
 685                 reset_control_deassert(pservice->rst_a);
 686                 if (pservice->rst_v)
 687                         reset_control_deassert(pservice->rst_v);
 688         }
 689 #endif
 690
 691         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 692                 clear_bit(MMU_ACTIVATED, &data->state);
 693                 if (atomic_read(&pservice->enabled))
 694                         rockchip_iovmm_deactivate(data->dev);
 695                 else
 696                         BUG_ON(!atomic_read(&pservice->enabled));
 697         }
 698
 699         atomic_set(&pservice->reset_request, 0);
 700         pr_info("done\n");
 701 }
 702
 703 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 704 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 705                                       struct vpu_session *session)
 706 {
 707         struct vpu_reg *reg, *n;
 708
 709         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 710                 reg_deinit(data, reg);
 711         }
 712         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 713                 reg_deinit(data, reg);
 714         }
 715         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 716                 reg_deinit(data, reg);
 717         }
 718 }
 719
 720 static void vpu_service_clear(struct vpu_subdev_data *data)
 721 {
 722         struct vpu_reg *reg, *n;
 723         struct vpu_session *session, *s;
 724         struct vpu_service_info *pservice = data->pservice;
 725
 726         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 727                 reg_deinit(data, reg);
 728         }
 729
 730         /* wake up session wait event to prevent the timeout hw reset
 731          * during reboot procedure.
 732          */
 733         list_for_each_entry_safe(session, s,
 734                                  &pservice->session, list_session)
 735                 wake_up(&session->wait);
 736 }
 737
 738 static void vpu_service_dump(struct vpu_service_info *pservice)
 739 {
 740 }
 741
 742
 743 static void vpu_service_power_off(struct vpu_service_info *pservice)
 744 {
 745         int total_running;
 746         struct vpu_subdev_data *data = NULL, *n;
 747         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 748
 749         if (!ret)
 750                 return;
 751
 752         total_running = atomic_read(&pservice->total_running);
 753         if (total_running) {
 754                 pr_alert("alert: power off when %d task running!!\n",
 755                          total_running);
 756                 mdelay(50);
 757                 pr_alert("alert: delay 50 ms for running task\n");
 758                 vpu_service_dump(pservice);
 759         }
 760
 761         pr_info("%s: power off...", dev_name(pservice->dev));
 762
 763         udelay(5);
 764
 765         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 766                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 767                         clear_bit(MMU_ACTIVATED, &data->state);
 768                         rockchip_iovmm_deactivate(data->dev);
 769                 }
 770         }
 771         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 772
 773 #if VCODEC_CLOCK_ENABLE
 774                 if (pservice->pd_video)
 775                         clk_disable_unprepare(pservice->pd_video);
 776                 if (pservice->hclk_vcodec)
 777                         clk_disable_unprepare(pservice->hclk_vcodec);
 778                 if (pservice->aclk_vcodec)
 779                         clk_disable_unprepare(pservice->aclk_vcodec);
 780                 if (pservice->clk_core)
 781                         clk_disable_unprepare(pservice->clk_core);
 782                 if (pservice->clk_cabac)
 783                         clk_disable_unprepare(pservice->clk_cabac);
 784 #endif
 785         pm_runtime_put(pservice->dev);
 786
 787         atomic_add(1, &pservice->power_off_cnt);
 788         wake_unlock(&pservice->wake_lock);
 789         pr_info("done\n");
 790 }
 791
 792 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 793 {
 794         queue_delayed_work(system_wq, &pservice->power_off_work,
 795                            VPU_POWER_OFF_DELAY);
 796 }
 797
 798 static void vpu_power_off_work(struct work_struct *work_s)
 799 {
 800         struct delayed_work *dlwork = container_of(work_s,
 801                         struct delayed_work, work);
 802         struct vpu_service_info *pservice = container_of(dlwork,
 803                         struct vpu_service_info, power_off_work);
 804
 805         if (mutex_trylock(&pservice->lock)) {
 806                 vpu_service_power_off(pservice);
 807                 mutex_unlock(&pservice->lock);
 808         } else {
 809                 /* Come back later if the device is busy... */
 810                 vpu_queue_power_off_work(pservice);
 811         }
 812 }
 813
 814 static void vpu_service_power_on(struct vpu_service_info *pservice)
 815 {
 816         int ret;
 817         ktime_t now = ktime_get();
 818
 819         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC) {
 820                 cancel_delayed_work_sync(&pservice->power_off_work);
 821                 vpu_queue_power_off_work(pservice);
 822                 pservice->last = now;
 823         }
 824         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 825         if (!ret)
 826                 return;
 827
 828         pr_info("%s: power on\n", dev_name(pservice->dev));
 829
 830 #define BIT_VCODEC_CLK_SEL      (1<<10)
 831         if (cpu_is_rk312x())
 832                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 833                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 834                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 835
 836 #if VCODEC_CLOCK_ENABLE
 837         if (pservice->aclk_vcodec)
 838                 clk_prepare_enable(pservice->aclk_vcodec);
 839         if (pservice->hclk_vcodec)
 840                 clk_prepare_enable(pservice->hclk_vcodec);
 841         if (pservice->clk_core)
 842                 clk_prepare_enable(pservice->clk_core);
 843         if (pservice->clk_cabac)
 844                 clk_prepare_enable(pservice->clk_cabac);
 845         if (pservice->pd_video)
 846                 clk_prepare_enable(pservice->pd_video);
 847 #endif
 848         pm_runtime_get_sync(pservice->dev);
 849
 850         udelay(5);
 851         atomic_add(1, &pservice->power_on_cnt);
 852         wake_lock(&pservice->wake_lock);
 853 }
 854
 855 static inline bool reg_check_interlace(struct vpu_reg *reg)
 856 {
 857         u32 type = (reg->reg[3] & (1 << 23));
 858
 859         return (type > 0);
 860 }
 861
 862 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 863 {
 864         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 865
 866         return type;
 867 }
 868
 869 static inline int reg_probe_width(struct vpu_reg *reg)
 870 {
 871         int width_in_mb = reg->reg[4] >> 23;
 872
 873         return width_in_mb * 16;
 874 }
 875
 876 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 877 {
 878         int y_virstride = reg->reg[8];
 879
 880         return y_virstride;
 881 }
 882
 883 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 884                              struct vpu_reg *reg, int fd)
 885 {
 886         struct vpu_service_info *pservice = data->pservice;
 887         struct ion_handle *hdl;
 888         int ret = 0;
 889         struct vcodec_mem_region *mem_region;
 890
 891         hdl = ion_import_dma_buf(pservice->ion_client, fd);
 892         if (IS_ERR(hdl)) {
 893                 vpu_err("import dma-buf from fd %d failed\n", fd);
 894                 return PTR_ERR(hdl);
 895         }
 896         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 897
 898         if (mem_region == NULL) {
 899                 vpu_err("allocate memory for iommu memory region failed\n");
 900                 ion_free(pservice->ion_client, hdl);
 901                 return -1;
 902         }
 903
 904         mem_region->hdl = hdl;
 905         if (data->mmu_dev)
 906                 ret = ion_map_iommu(data->dev, pservice->ion_client,
 907                                     mem_region->hdl, &mem_region->iova,
 908                                     &mem_region->len);
 909         else
 910                 ret = ion_phys(pservice->ion_client,
 911                                mem_region->hdl,
 912                                (ion_phys_addr_t *)&mem_region->iova,
 913                                (size_t *)&mem_region->len);
 914
 915         if (ret < 0) {
 916                 vpu_err("fd %d ion map iommu failed\n", fd);
 917                 kfree(mem_region);
 918                 ion_free(pservice->ion_client, hdl);
 919                 return ret;
 920         }
 921         INIT_LIST_HEAD(&mem_region->reg_lnk);
 922         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 923         return mem_region->iova;
 924 }
 925
 926 /*
 927  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 928  * it by pps id in video stream data.
 929  *
 930  * So we need to translate the address in iommu case. The address data is also
 931  * 10bit fd + 22bit offset mode.
 932  * Because userspace decoder do not give the pps id in the register file sets
 933  * kernel driver need to translate each scaling list address in pps buffer which
 934  * means 256 pps for H.264, 64 pps for H.265.
 935  *
 936  * In order to optimize the performance kernel driver ask userspace decoder to
 937  * set all scaling list address in pps buffer to the same one which will be used
 938  * on current decoding task. Then kernel driver can only translate the first
 939  * address then copy it all pps buffer.
 940  */
 941 static void fill_scaling_list_addr_in_pps(
 942                 struct vpu_subdev_data *data,
 943                 struct vpu_reg *reg,
 944                 char *pps,
 945                 int pps_info_count,
 946                 int pps_info_size,
 947                 int scaling_list_addr_offset)
 948 {
 949         int base = scaling_list_addr_offset;
 950         int scaling_fd = 0;
 951         u32 scaling_offset;
 952
 953         scaling_offset  = (u32)pps[base + 0];
 954         scaling_offset += (u32)pps[base + 1] << 8;
 955         scaling_offset += (u32)pps[base + 2] << 16;
 956         scaling_offset += (u32)pps[base + 3] << 24;
 957
 958         scaling_fd = scaling_offset & 0x3ff;
 959         scaling_offset = scaling_offset >> 10;
 960
 961         if (scaling_fd > 0) {
 962                 int i = 0;
 963                 u32 tmp = vcodec_fd_to_iova(data, reg, scaling_fd);
 964                 tmp += scaling_offset;
 965
 966                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 967                         pps[base + 0] = (tmp >>  0) & 0xff;
 968                         pps[base + 1] = (tmp >>  8) & 0xff;
 969                         pps[base + 2] = (tmp >> 16) & 0xff;
 970                         pps[base + 3] = (tmp >> 24) & 0xff;
 971                 }
 972         }
 973 }
 974
 975 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data, const u8 *tbl,
 976                                 int size, struct vpu_reg *reg,
 977                                 struct extra_info_for_iommu *ext_inf)
 978 {
 979         struct vpu_service_info *pservice = data->pservice;
 980         struct vpu_task_info *task = reg->task;
 981         enum FORMAT_TYPE type;
 982         struct ion_handle *hdl;
 983         int ret = 0;
 984         struct vcodec_mem_region *mem_region;
 985         int i;
 986         int offset = 0;
 987
 988         if (tbl == NULL || size <= 0) {
 989                 dev_err(pservice->dev, "input arguments invalidate\n");
 990                 return -1;
 991         }
 992
 993         if (task->get_fmt)
 994                 type = task->get_fmt(reg->reg);
 995         else {
 996                 pr_err("invalid task with NULL get_fmt\n");
 997                 return -1;
 998         }
 999
1000         for (i = 0; i < size; i++) {
1001                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
1002
1003                 /* if userspace do not set the fd at this register, skip */
1004                 if (usr_fd == 0)
1005                         continue;
1006
1007                 /*
1008                  * special offset scale case
1009                  *
1010                  * This translation is for fd + offset translation.
1011                  * One register has 32bits. We need to transfer both buffer file
1012                  * handle and the start address offset so we packet file handle
1013                  * and offset together using below format.
1014                  *
1015                  *  0~9  bit for buffer file handle range 0 ~ 1023
1016                  * 10~31 bit for offset range 0 ~ 4M
1017                  *
1018                  * But on 4K case the offset can be larger the 4M
1019                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1020                  * But MPEG4 will use the same register for colmv and it do not
1021                  * need scale.
1022                  *
1023                  * RKVdec do not have this issue.
1024                  */
1025                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1026                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1027                         offset = reg->reg[tbl[i]] >> 10 << 4;
1028                 else
1029                         offset = reg->reg[tbl[i]] >> 10;
1030
1031                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d\n",
1032                           tbl[i], usr_fd, offset);
1033
1034                 hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
1035                 if (IS_ERR(hdl)) {
1036                         dev_err(pservice->dev,
1037                                 "import dma-buf from fd %d failed, reg[%d]\n",
1038                                 usr_fd, tbl[i]);
1039                         return PTR_ERR(hdl);
1040                 }
1041
1042                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1043                         int pps_info_offset;
1044                         int pps_info_count;
1045                         int pps_info_size;
1046                         int scaling_list_addr_offset;
1047
1048                         switch (type) {
1049                         case FMT_H264D: {
1050                                 pps_info_offset = offset;
1051                                 pps_info_count = 256;
1052                                 pps_info_size = 32;
1053                                 scaling_list_addr_offset = 23;
1054                         } break;
1055                         case FMT_H265D: {
1056                                 pps_info_offset = 0;
1057                                 pps_info_count = 64;
1058                                 pps_info_size = 80;
1059                                 scaling_list_addr_offset = 74;
1060                         } break;
1061                         default: {
1062                                 pps_info_offset = 0;
1063                                 pps_info_count = 0;
1064                                 pps_info_size = 0;
1065                                 scaling_list_addr_offset = 0;
1066                         } break;
1067                         }
1068
1069                         vpu_debug(DEBUG_PPS_FILL,
1070                                   "scaling list filling parameter:\n");
1071                         vpu_debug(DEBUG_PPS_FILL,
1072                                   "pps_info_offset %d\n", pps_info_offset);
1073                         vpu_debug(DEBUG_PPS_FILL,
1074                                   "pps_info_count  %d\n", pps_info_count);
1075                         vpu_debug(DEBUG_PPS_FILL,
1076                                   "pps_info_size   %d\n", pps_info_size);
1077                         vpu_debug(DEBUG_PPS_FILL,
1078                                   "scaling_list_addr_offset %d\n",
1079                                   scaling_list_addr_offset);
1080
1081                         if (pps_info_count) {
1082                                 char *pps = (char *)ion_map_kernel(
1083                                                 pservice->ion_client, hdl);
1084                                 vpu_debug(DEBUG_PPS_FILL,
1085                                           "scaling list setting pps %p\n", pps);
1086                                 pps += pps_info_offset;
1087
1088                                 fill_scaling_list_addr_in_pps(
1089                                                 data, reg, pps,
1090                                                 pps_info_count,
1091                                                 pps_info_size,
1092                                                 scaling_list_addr_offset);
1093                         }
1094                 }
1095
1096                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1097
1098                 if (!mem_region) {
1099                         ion_free(pservice->ion_client, hdl);
1100                         return -1;
1101                 }
1102
1103                 mem_region->hdl = hdl;
1104                 mem_region->reg_idx = tbl[i];
1105
1106                 if (data->mmu_dev)
1107                         ret = ion_map_iommu(data->dev,
1108                                             pservice->ion_client,
1109                                             mem_region->hdl,
1110                                             &mem_region->iova,
1111                                             &mem_region->len);
1112                 else
1113                         ret = ion_phys(pservice->ion_client,
1114                                        mem_region->hdl,
1115                                        (ion_phys_addr_t *)&mem_region->iova,
1116                                        (size_t *)&mem_region->len);
1117
1118                 if (ret < 0) {
1119                         dev_err(pservice->dev, "reg %d fd %d ion map iommu failed\n",
1120                                 tbl[i], usr_fd);
1121                         kfree(mem_region);
1122                         ion_free(pservice->ion_client, hdl);
1123                         return ret;
1124                 }
1125
1126                 /*
1127                  * special for vpu dec num 12: record decoded length
1128                  * hacking for decoded length
1129                  * NOTE: not a perfect fix, the fd is not recorded
1130                  */
1131                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1132                         reg->dec_base = mem_region->iova + offset;
1133                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1134                                   reg->dec_base);
1135                 }
1136
1137                 reg->reg[tbl[i]] = mem_region->iova + offset;
1138                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1139                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1140         }
1141
1142         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1143                 for (i = 0; i < ext_inf->cnt; i++) {
1144                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1145                                   ext_inf->elem[i].index,
1146                                   ext_inf->elem[i].offset);
1147                         reg->reg[ext_inf->elem[i].index] +=
1148                                 ext_inf->elem[i].offset;
1149                 }
1150         }
1151
1152         return 0;
1153 }
1154
1155 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1156                                         struct vpu_reg *reg,
1157                                         struct extra_info_for_iommu *ext_inf)
1158 {
1159         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1160
1161         if (type < FMT_TYPE_BUTT) {
1162                 const struct vpu_trans_info *info = &reg->trans[type];
1163                 const u8 *tbl = info->table;
1164                 int size = info->count;
1165
1166                 return vcodec_bufid_to_iova(data, tbl, size, reg, ext_inf);
1167         }
1168         pr_err("found invalid format type!\n");
1169         return -1;
1170 }
1171
1172 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1173 {
1174
1175         if (!soc_is_rk2928g()) {
1176                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1177                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1178                                 if (reg_probe_width(reg) > 3200) {
1179                                         /*raise frequency for 4k avc.*/
1180                                         reg->freq = VPU_FREQ_600M;
1181                                 }
1182                         } else {
1183                                 if (reg_check_interlace(reg))
1184                                         reg->freq = VPU_FREQ_400M;
1185                         }
1186                 }
1187                 if (data->hw_id == HEVC_ID) {
1188                         if (reg_probe_hevc_y_stride(reg) > 60000)
1189                                 reg->freq = VPU_FREQ_400M;
1190                 }
1191                 if (reg->type == VPU_PP)
1192                         reg->freq = VPU_FREQ_400M;
1193         }
1194 }
1195
1196 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1197                                 struct vpu_session *session,
1198                                 void __user *src, u32 size)
1199 {
1200         struct vpu_service_info *pservice = data->pservice;
1201         int extra_size = 0;
1202         struct extra_info_for_iommu extra_info;
1203         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1204                                       GFP_KERNEL);
1205
1206         vpu_debug_enter();
1207
1208         if (NULL == reg) {
1209                 vpu_err("error: kmalloc fail in reg_init\n");
1210                 return NULL;
1211         }
1212
1213         if (size > data->reg_size) {
1214                 extra_size = size - data->reg_size;
1215                 size = data->reg_size;
1216         }
1217         reg->session = session;
1218         reg->data = data;
1219         reg->type = session->type;
1220         reg->size = size;
1221         reg->freq = VPU_FREQ_DEFAULT;
1222         reg->task = &data->task_info[session->type];
1223         reg->trans = data->trans_info;
1224         reg->reg = (u32 *)&reg[1];
1225         INIT_LIST_HEAD(&reg->session_link);
1226         INIT_LIST_HEAD(&reg->status_link);
1227
1228         INIT_LIST_HEAD(&reg->mem_region_list);
1229
1230         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1231                 vpu_err("error: copy_from_user failed in reg_init\n");
1232                 kfree(reg);
1233                 return NULL;
1234         }
1235
1236         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1237                 vpu_err("error: copy_from_user failed in reg_init\n");
1238                 kfree(reg);
1239                 return NULL;
1240         }
1241
1242         if (0 > vcodec_reg_address_translate(data, reg, &extra_info)) {
1243                 int i = 0;
1244
1245                 vpu_err("error: translate reg address failed, dumping regs\n");
1246                 for (i = 0; i < size >> 2; i++)
1247                         pr_err("reg[%02d]: %08x\n", i, *((u32 *)src + i));
1248
1249                 kfree(reg);
1250                 return NULL;
1251         }
1252
1253         mutex_lock(&pservice->lock);
1254         list_add_tail(&reg->status_link, &pservice->waiting);
1255         list_add_tail(&reg->session_link, &session->waiting);
1256         mutex_unlock(&pservice->lock);
1257
1258         if (pservice->auto_freq)
1259                 get_reg_freq(data, reg);
1260
1261         vpu_debug_leave();
1262         return reg;
1263 }
1264
1265 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1266 {
1267         struct vpu_service_info *pservice = data->pservice;
1268         struct vcodec_mem_region *mem_region = NULL, *n;
1269
1270         list_del_init(&reg->session_link);
1271         list_del_init(&reg->status_link);
1272         if (reg == pservice->reg_codec)
1273                 pservice->reg_codec = NULL;
1274         if (reg == pservice->reg_pproc)
1275                 pservice->reg_pproc = NULL;
1276
1277         /* release memory region attach to this registers table. */
1278         list_for_each_entry_safe(mem_region, n,
1279                         &reg->mem_region_list, reg_lnk) {
1280                 ion_free(pservice->ion_client, mem_region->hdl);
1281                 list_del_init(&mem_region->reg_lnk);
1282                 kfree(mem_region);
1283         }
1284
1285         kfree(reg);
1286 }
1287
1288 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1289                                  struct vpu_reg *reg)
1290 {
1291         vpu_debug_enter();
1292         list_del_init(&reg->status_link);
1293         list_add_tail(&reg->status_link, &pservice->running);
1294
1295         list_del_init(&reg->session_link);
1296         list_add_tail(&reg->session_link, &reg->session->running);
1297         vpu_debug_leave();
1298 }
1299
1300 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1301 {
1302         int i;
1303         u32 *dst = reg->reg;
1304
1305         vpu_debug_enter();
1306         for (i = 0; i < count; i++, src++)
1307                 *dst++ = readl_relaxed(src);
1308
1309         dst = (u32 *)&reg->reg[0];
1310         for (i = 0; i < count; i++)
1311                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1312
1313         vpu_debug_leave();
1314 }
1315
1316 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1317                                  struct vpu_reg *reg)
1318 {
1319         struct vpu_service_info *pservice = data->pservice;
1320         struct vpu_hw_info *hw_info = data->hw_info;
1321         struct vpu_task_info *task = reg->task;
1322
1323         vpu_debug_enter();
1324
1325         list_del_init(&reg->status_link);
1326         list_add_tail(&reg->status_link, &pservice->done);
1327
1328         list_del_init(&reg->session_link);
1329         list_add_tail(&reg->session_link, &reg->session->done);
1330
1331         switch (reg->type) {
1332         case VPU_ENC: {
1333                 pservice->reg_codec = NULL;
1334                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1335                 reg->reg[task->reg_irq] = pservice->irq_status;
1336         } break;
1337         case VPU_DEC: {
1338                 pservice->reg_codec = NULL;
1339                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1340
1341                 /* revert hack for decoded length */
1342                 if (task->reg_len > 0) {
1343                         int reg_len = task->reg_len;
1344                         u32 dec_get = reg->reg[reg_len];
1345                         s32 dec_length = dec_get - reg->dec_base;
1346
1347                         vpu_debug(DEBUG_REGISTER,
1348                                   "dec_get %08x dec_length %d\n",
1349                                   dec_get, dec_length);
1350                         reg->reg[reg_len] = dec_length << 10;
1351                 }
1352
1353                 reg->reg[task->reg_irq] = pservice->irq_status;
1354         } break;
1355         case VPU_PP: {
1356                 pservice->reg_pproc = NULL;
1357                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1358                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1359         } break;
1360         case VPU_DEC_PP: {
1361                 u32 pipe_mode;
1362                 u32 *regs = data->dec_dev.regs;
1363
1364                 pservice->reg_codec = NULL;
1365                 pservice->reg_pproc = NULL;
1366
1367                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1368
1369                 /* NOTE: remove pp pipeline mode flag first */
1370                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1371                 pipe_mode &= ~task->pipe_mask;
1372                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1373
1374                 /* revert hack for decoded length */
1375                 if (task->reg_len > 0) {
1376                         int reg_len = task->reg_len;
1377                         u32 dec_get = reg->reg[reg_len];
1378                         s32 dec_length = dec_get - reg->dec_base;
1379
1380                         vpu_debug(DEBUG_REGISTER,
1381                                   "dec_get %08x dec_length %d\n",
1382                                   dec_get, dec_length);
1383                         reg->reg[reg_len] = dec_length << 10;
1384                 }
1385
1386                 reg->reg[task->reg_irq] = pservice->irq_status;
1387         } break;
1388         default: {
1389                 vpu_err("error: copy reg from hw with unknown type %d\n",
1390                         reg->type);
1391         } break;
1392         }
1393         vcodec_exit_mode(data);
1394
1395         atomic_sub(1, &reg->session->task_running);
1396         atomic_sub(1, &pservice->total_running);
1397         wake_up(&reg->session->wait);
1398
1399         vpu_debug_leave();
1400 }
1401
1402 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1403                                  struct vpu_reg *reg)
1404 {
1405         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1406
1407         if (curr == reg->freq)
1408                 return;
1409
1410         atomic_set(&pservice->freq_status, reg->freq);
1411         switch (reg->freq) {
1412         case VPU_FREQ_200M: {
1413                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1414         } break;
1415         case VPU_FREQ_266M: {
1416                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1417         } break;
1418         case VPU_FREQ_300M: {
1419                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1420         } break;
1421         case VPU_FREQ_400M: {
1422                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1423         } break;
1424         case VPU_FREQ_500M: {
1425                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1426         } break;
1427         case VPU_FREQ_600M: {
1428                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1429         } break;
1430         default: {
1431                 unsigned long rate = 300*MHZ;
1432
1433                 if (soc_is_rk2928g())
1434                         rate = 400*MHZ;
1435
1436                 clk_set_rate(pservice->aclk_vcodec, rate);
1437         } break;
1438         }
1439 }
1440
1441 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1442 {
1443         struct vpu_service_info *pservice = data->pservice;
1444         struct vpu_task_info *task = reg->task;
1445         struct vpu_hw_info *hw_info = data->hw_info;
1446         int i;
1447         u32 *src = (u32 *)&reg->reg[0];
1448         u32 enable_mask = task->enable_mask;
1449         u32 gating_mask = task->gating_mask;
1450         u32 reg_en = task->reg_en;
1451
1452         vpu_debug_enter();
1453
1454         atomic_add(1, &pservice->total_running);
1455         atomic_add(1, &reg->session->task_running);
1456
1457         if (pservice->auto_freq)
1458                 vpu_service_set_freq(pservice, reg);
1459
1460         vcodec_enter_mode(data);
1461
1462         switch (reg->type) {
1463         case VPU_ENC: {
1464                 u32 *dst = data->enc_dev.regs;
1465                 u32 base = 0;
1466                 u32 end  = hw_info->enc_reg_num;
1467                 /* u32 reg_gating = task->reg_gating; */
1468
1469                 pservice->reg_codec = reg;
1470
1471                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1472                           base, end, reg_en, enable_mask, gating_mask);
1473
1474                 VEPU_CLEAN_CACHE(dst);
1475
1476                 if (debug & DEBUG_SET_REG)
1477                         for (i = base; i < end; i++)
1478                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1479                                           i, src[i]);
1480
1481                 /*
1482                  * NOTE: encoder need to setup mode first
1483                  */
1484                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1485
1486                 /* NOTE: encoder gating is not on enable register */
1487                 /* src[reg_gating] |= gating_mask; */
1488
1489                 for (i = base; i < end; i++) {
1490                         if (i != reg_en)
1491                                 writel_relaxed(src[i], dst + i);
1492                 }
1493
1494                 writel(src[reg_en], dst + reg_en);
1495                 dsb(sy);
1496
1497                 time_record(reg->task, 0);
1498         } break;
1499         case VPU_DEC: {
1500                 u32 *dst = data->dec_dev.regs;
1501                 u32 len = hw_info->dec_reg_num;
1502                 u32 base = hw_info->base_dec;
1503                 u32 end  = hw_info->end_dec;
1504
1505                 pservice->reg_codec = reg;
1506
1507                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1508                           base, end, reg_en, enable_mask, gating_mask);
1509
1510                 VDPU_CLEAN_CACHE(dst);
1511
1512                 /* on rkvdec set cache size to 64byte */
1513                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1514                         u32 *cache_base = dst + 0x100;
1515                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1516                         writel_relaxed(val, cache_base + 0x07);
1517                         writel_relaxed(val, cache_base + 0x17);
1518                 }
1519
1520                 if (debug & DEBUG_SET_REG)
1521                         for (i = 0; i < len; i++)
1522                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1523                                           i, src[i]);
1524
1525                 /*
1526                  * NOTE: The end register is invalid. Do NOT write to it
1527                  *       Also the base register must be written
1528                  */
1529                 for (i = base; i < end; i++) {
1530                         if (i != reg_en)
1531                                 writel_relaxed(src[i], dst + i);
1532                 }
1533
1534                 writel(src[reg_en] | gating_mask, dst + reg_en);
1535                 dsb(sy);
1536
1537                 time_record(reg->task, 0);
1538         } break;
1539         case VPU_PP: {
1540                 u32 *dst = data->dec_dev.regs;
1541                 u32 base = hw_info->base_pp;
1542                 u32 end  = hw_info->end_pp;
1543
1544                 pservice->reg_pproc = reg;
1545
1546                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1547                           base, end, reg_en, enable_mask, gating_mask);
1548
1549                 if (debug & DEBUG_SET_REG)
1550                         for (i = base; i < end; i++)
1551                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1552                                           i, src[i]);
1553
1554                 for (i = base; i < end; i++) {
1555                         if (i != reg_en)
1556                                 writel_relaxed(src[i], dst + i);
1557                 }
1558
1559                 writel(src[reg_en] | gating_mask, dst + reg_en);
1560                 dsb(sy);
1561
1562                 time_record(reg->task, 0);
1563         } break;
1564         case VPU_DEC_PP: {
1565                 u32 *dst = data->dec_dev.regs;
1566                 u32 base = hw_info->base_dec_pp;
1567                 u32 end  = hw_info->end_dec_pp;
1568
1569                 pservice->reg_codec = reg;
1570                 pservice->reg_pproc = reg;
1571
1572                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1573                           base, end, reg_en, enable_mask, gating_mask);
1574
1575                 /* VDPU_SOFT_RESET(dst); */
1576                 VDPU_CLEAN_CACHE(dst);
1577
1578                 if (debug & DEBUG_SET_REG)
1579                         for (i = base; i < end; i++)
1580                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1581                                           i, src[i]);
1582
1583                 for (i = base; i < end; i++) {
1584                         if (i != reg_en)
1585                                 writel_relaxed(src[i], dst + i);
1586                 }
1587
1588                 /* NOTE: dec output must be disabled */
1589
1590                 writel(src[reg_en] | gating_mask, dst + reg_en);
1591                 dsb(sy);
1592
1593                 time_record(reg->task, 0);
1594         } break;
1595         default: {
1596                 vpu_err("error: unsupport session type %d", reg->type);
1597                 atomic_sub(1, &pservice->total_running);
1598                 atomic_sub(1, &reg->session->task_running);
1599         } break;
1600         }
1601
1602         vpu_debug_leave();
1603 }
1604
1605 static void try_set_reg(struct vpu_subdev_data *data)
1606 {
1607         struct vpu_service_info *pservice = data->pservice;
1608
1609         vpu_debug_enter();
1610
1611         mutex_lock(&pservice->shutdown_lock);
1612         if (atomic_read(&pservice->service_on) == 0) {
1613                 mutex_lock(&pservice->shutdown_lock);
1614                 return;
1615         }
1616         if (!list_empty(&pservice->waiting)) {
1617                 struct vpu_reg *reg_codec = pservice->reg_codec;
1618                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1619                 int can_set = 0;
1620                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1621                 int reset_request = atomic_read(&pservice->reset_request);
1622                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1623                                 struct vpu_reg, status_link);
1624
1625                 vpu_service_power_on(pservice);
1626
1627                 if (change_able || !reset_request) {
1628                         switch (reg->type) {
1629                         case VPU_ENC: {
1630                                 if (change_able)
1631                                         can_set = 1;
1632                         } break;
1633                         case VPU_DEC: {
1634                                 if (reg_codec == NULL)
1635                                         can_set = 1;
1636                                 if (pservice->auto_freq && (reg_pproc != NULL))
1637                                         can_set = 0;
1638                         } break;
1639                         case VPU_PP: {
1640                                 if (reg_codec == NULL) {
1641                                         if (reg_pproc == NULL)
1642                                                 can_set = 1;
1643                                 } else {
1644                                         if ((reg_codec->type == VPU_DEC) &&
1645                                             (reg_pproc == NULL))
1646                                                 can_set = 1;
1647
1648                                         /*
1649                                          * NOTE:
1650                                          * can not charge frequency
1651                                          * when vpu is working
1652                                          */
1653                                         if (pservice->auto_freq)
1654                                                 can_set = 0;
1655                                 }
1656                         } break;
1657                         case VPU_DEC_PP: {
1658                                 if (change_able)
1659                                         can_set = 1;
1660                                 } break;
1661                         default: {
1662                                 pr_err("undefined reg type %d\n", reg->type);
1663                         } break;
1664                         }
1665                 }
1666
1667                 /* then check reset request */
1668                 if (reset_request && !change_able)
1669                         reset_request = 0;
1670
1671                 /* do reset before setting registers */
1672                 if (reset_request)
1673                         vpu_reset(data);
1674
1675                 if (can_set) {
1676                         reg_from_wait_to_run(pservice, reg);
1677                         reg_copy_to_hw(reg->data, reg);
1678                 }
1679         }
1680
1681         mutex_unlock(&pservice->shutdown_lock);
1682         vpu_debug_leave();
1683 }
1684
1685 static int return_reg(struct vpu_subdev_data *data,
1686                       struct vpu_reg *reg, u32 __user *dst)
1687 {
1688         struct vpu_hw_info *hw_info = data->hw_info;
1689         size_t size = reg->size;
1690         u32 base;
1691
1692         vpu_debug_enter();
1693         switch (reg->type) {
1694         case VPU_ENC: {
1695                 base = 0;
1696         } break;
1697         case VPU_DEC: {
1698                 base = hw_info->base_dec_pp;
1699         } break;
1700         case VPU_PP: {
1701                 base = hw_info->base_pp;
1702         } break;
1703         case VPU_DEC_PP: {
1704                 base = hw_info->base_dec_pp;
1705         } break;
1706         default: {
1707                 vpu_err("error: copy reg to user with unknown type %d\n",
1708                         reg->type);
1709                 return -EFAULT;
1710         } break;
1711         }
1712
1713         if (copy_to_user(dst, &reg->reg[base], size)) {
1714                 vpu_err("error: return_reg copy_to_user failed\n");
1715                 return -EFAULT;
1716         }
1717
1718         reg_deinit(data, reg);
1719         vpu_debug_leave();
1720         return 0;
1721 }
1722
1723 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1724                               unsigned long arg)
1725 {
1726         struct vpu_subdev_data *data =
1727                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1728                              struct vpu_subdev_data, cdev);
1729         struct vpu_service_info *pservice = data->pservice;
1730         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1731
1732         vpu_debug_enter();
1733         if (NULL == session)
1734                 return -EINVAL;
1735
1736         switch (cmd) {
1737         case VPU_IOC_SET_CLIENT_TYPE: {
1738                 session->type = (enum VPU_CLIENT_TYPE)arg;
1739                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1740                           session->pid, session->type);
1741         } break;
1742         case VPU_IOC_GET_HW_FUSE_STATUS: {
1743                 struct vpu_request req;
1744
1745                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1746                           session->pid, session->type);
1747                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1748                         vpu_err("error: get hw status copy_from_user failed\n");
1749                         return -EFAULT;
1750                 } else {
1751                         void *config = (session->type != VPU_ENC) ?
1752                                        ((void *)&pservice->dec_config) :
1753                                        ((void *)&pservice->enc_config);
1754                         size_t size = (session->type != VPU_ENC) ?
1755                                       (sizeof(struct vpu_dec_config)) :
1756                                       (sizeof(struct vpu_enc_config));
1757                         if (copy_to_user((void __user *)req.req,
1758                                          config, size)) {
1759                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1760                                         session->type);
1761                                 return -EFAULT;
1762                         }
1763                 }
1764         } break;
1765         case VPU_IOC_SET_REG: {
1766                 struct vpu_request req;
1767                 struct vpu_reg *reg;
1768
1769                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1770                           session->pid, session->type);
1771                 if (copy_from_user(&req, (void __user *)arg,
1772                                    sizeof(struct vpu_request))) {
1773                         vpu_err("error: set reg copy_from_user failed\n");
1774                         return -EFAULT;
1775                 }
1776
1777                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1778                 if (NULL == reg) {
1779                         return -EFAULT;
1780                 } else {
1781                         mutex_lock(&pservice->lock);
1782                         try_set_reg(data);
1783                         mutex_unlock(&pservice->lock);
1784                 }
1785         } break;
1786         case VPU_IOC_GET_REG: {
1787                 struct vpu_request req;
1788                 struct vpu_reg *reg;
1789                 int ret;
1790
1791                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1792                           session->pid, session->type);
1793                 if (copy_from_user(&req, (void __user *)arg,
1794                                    sizeof(struct vpu_request))) {
1795                         vpu_err("error: get reg copy_from_user failed\n");
1796                         return -EFAULT;
1797                 }
1798
1799                 ret = wait_event_timeout(session->wait,
1800                                          !list_empty(&session->done),
1801                                          VPU_TIMEOUT_DELAY);
1802
1803                 if (!list_empty(&session->done)) {
1804                         if (ret < 0)
1805                                 vpu_err("warning: pid %d wait task error ret %d\n",
1806                                         session->pid, ret);
1807                         ret = 0;
1808                 } else {
1809                         if (unlikely(ret < 0)) {
1810                                 vpu_err("error: pid %d wait task ret %d\n",
1811                                         session->pid, ret);
1812                         } else if (ret == 0) {
1813                                 vpu_err("error: pid %d wait %d task done timeout\n",
1814                                         session->pid,
1815                                         atomic_read(&session->task_running));
1816                                 ret = -ETIMEDOUT;
1817                         }
1818                 }
1819
1820                 if (ret < 0) {
1821                         int task_running = atomic_read(&session->task_running);
1822
1823                         mutex_lock(&pservice->lock);
1824                         vpu_service_dump(pservice);
1825                         if (task_running) {
1826                                 atomic_set(&session->task_running, 0);
1827                                 atomic_sub(task_running,
1828                                            &pservice->total_running);
1829                                 pr_err("%d task is running but not return, reset hardware...",
1830                                        task_running);
1831                                 vpu_reset(data);
1832                                 pr_err("done\n");
1833                         }
1834                         vpu_service_session_clear(data, session);
1835                         mutex_unlock(&pservice->lock);
1836                         return ret;
1837                 }
1838
1839                 mutex_lock(&pservice->lock);
1840                 reg = list_entry(session->done.next,
1841                                  struct vpu_reg, session_link);
1842                 return_reg(data, reg, (u32 __user *)req.req);
1843                 mutex_unlock(&pservice->lock);
1844         } break;
1845         case VPU_IOC_PROBE_IOMMU_STATUS: {
1846                 int iommu_enable = 1;
1847
1848                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1849
1850                 if (copy_to_user((void __user *)arg,
1851                                  &iommu_enable, sizeof(int))) {
1852                         vpu_err("error: iommu status copy_to_user failed\n");
1853                         return -EFAULT;
1854                 }
1855         } break;
1856         default: {
1857                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1858         } break;
1859         }
1860         vpu_debug_leave();
1861         return 0;
1862 }
1863
1864 #ifdef CONFIG_COMPAT
1865 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1866                                      unsigned long arg)
1867 {
1868         struct vpu_subdev_data *data =
1869                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1870                              struct vpu_subdev_data, cdev);
1871         struct vpu_service_info *pservice = data->pservice;
1872         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1873
1874         vpu_debug_enter();
1875         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1876                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1877         if (NULL == session)
1878                 return -EINVAL;
1879
1880         switch (cmd) {
1881         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1882                 session->type = (enum VPU_CLIENT_TYPE)arg;
1883                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1884                           session->type);
1885         } break;
1886         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1887                 struct compat_vpu_request req;
1888
1889                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1890                           session->type);
1891                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1892                                    sizeof(struct compat_vpu_request))) {
1893                         vpu_err("error: compat get hw status copy_from_user failed\n");
1894                         return -EFAULT;
1895                 } else {
1896                         void *config = (session->type != VPU_ENC) ?
1897                                        ((void *)&pservice->dec_config) :
1898                                        ((void *)&pservice->enc_config);
1899                         size_t size = (session->type != VPU_ENC) ?
1900                                       (sizeof(struct vpu_dec_config)) :
1901                                       (sizeof(struct vpu_enc_config));
1902
1903                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1904                                          config, size)) {
1905                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1906                                         session->type);
1907                                 return -EFAULT;
1908                         }
1909                 }
1910         } break;
1911         case COMPAT_VPU_IOC_SET_REG: {
1912                 struct compat_vpu_request req;
1913                 struct vpu_reg *reg;
1914
1915                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1916                           session->type);
1917                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1918                                    sizeof(struct compat_vpu_request))) {
1919                         vpu_err("compat set_reg copy_from_user failed\n");
1920                         return -EFAULT;
1921                 }
1922                 reg = reg_init(data, session,
1923                                compat_ptr((compat_uptr_t)req.req), req.size);
1924                 if (NULL == reg) {
1925                         return -EFAULT;
1926                 } else {
1927                         mutex_lock(&pservice->lock);
1928                         try_set_reg(data);
1929                         mutex_unlock(&pservice->lock);
1930                 }
1931         } break;
1932         case COMPAT_VPU_IOC_GET_REG: {
1933                 struct compat_vpu_request req;
1934                 struct vpu_reg *reg;
1935                 int ret;
1936
1937                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1938                           session->type);
1939                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1940                                    sizeof(struct compat_vpu_request))) {
1941                         vpu_err("compat get reg copy_from_user failed\n");
1942                         return -EFAULT;
1943                 }
1944
1945                 ret = wait_event_timeout(session->wait,
1946                                          !list_empty(&session->done),
1947                                          VPU_TIMEOUT_DELAY);
1948
1949                 if (!list_empty(&session->done)) {
1950                         if (ret < 0)
1951                                 vpu_err("warning: pid %d wait task error ret %d\n",
1952                                         session->pid, ret);
1953                         ret = 0;
1954                 } else {
1955                         if (unlikely(ret < 0)) {
1956                                 vpu_err("error: pid %d wait task ret %d\n",
1957                                         session->pid, ret);
1958                         } else if (ret == 0) {
1959                                 vpu_err("error: pid %d wait %d task done timeout\n",
1960                                         session->pid,
1961                                         atomic_read(&session->task_running));
1962                                 ret = -ETIMEDOUT;
1963                         }
1964                 }
1965
1966                 if (ret < 0) {
1967                         int task_running = atomic_read(&session->task_running);
1968
1969                         mutex_lock(&pservice->lock);
1970                         vpu_service_dump(pservice);
1971                         if (task_running) {
1972                                 atomic_set(&session->task_running, 0);
1973                                 atomic_sub(task_running,
1974                                            &pservice->total_running);
1975                                 pr_err("%d task is running but not return, reset hardware...",
1976                                        task_running);
1977                                 vpu_reset(data);
1978                                 pr_err("done\n");
1979                         }
1980                         vpu_service_session_clear(data, session);
1981                         mutex_unlock(&pservice->lock);
1982                         return ret;
1983                 }
1984
1985                 mutex_lock(&pservice->lock);
1986                 reg = list_entry(session->done.next,
1987                                  struct vpu_reg, session_link);
1988                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
1989                 mutex_unlock(&pservice->lock);
1990         } break;
1991         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
1992                 int iommu_enable = 1;
1993
1994                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
1995
1996                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
1997                                  &iommu_enable, sizeof(int))) {
1998                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1999                         return -EFAULT;
2000                 }
2001         } break;
2002         default: {
2003                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2004         } break;
2005         }
2006         vpu_debug_leave();
2007         return 0;
2008 }
2009 #endif
2010
2011 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2012 {
2013         int ret = -EINVAL, i = 0;
2014         u32 hw_id = readl_relaxed(data->regs);
2015
2016         hw_id = (hw_id >> 16) & 0xFFFF;
2017         pr_info("checking hw id %x\n", hw_id);
2018         data->hw_info = NULL;
2019         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2020                 struct vcodec_info *info = &vcodec_info_set[i];
2021
2022                 if (hw_id == info->hw_id) {
2023                         data->hw_id = info->hw_id;
2024                         data->hw_info = info->hw_info;
2025                         data->task_info = info->task_info;
2026                         data->trans_info = info->trans_info;
2027                         ret = 0;
2028                         break;
2029                 }
2030         }
2031         return ret;
2032 }
2033
2034 static int vpu_service_open(struct inode *inode, struct file *filp)
2035 {
2036         struct vpu_subdev_data *data = container_of(
2037                         inode->i_cdev, struct vpu_subdev_data, cdev);
2038         struct vpu_service_info *pservice = data->pservice;
2039         struct vpu_session *session = kmalloc(sizeof(*session), GFP_KERNEL);
2040
2041         vpu_debug_enter();
2042
2043         if (NULL == session) {
2044                 vpu_err("error: unable to allocate memory for vpu_session.");
2045                 return -ENOMEM;
2046         }
2047
2048         session->type   = VPU_TYPE_BUTT;
2049         session->pid    = current->pid;
2050         INIT_LIST_HEAD(&session->waiting);
2051         INIT_LIST_HEAD(&session->running);
2052         INIT_LIST_HEAD(&session->done);
2053         INIT_LIST_HEAD(&session->list_session);
2054         init_waitqueue_head(&session->wait);
2055         atomic_set(&session->task_running, 0);
2056         mutex_lock(&pservice->lock);
2057         list_add_tail(&session->list_session, &pservice->session);
2058         filp->private_data = (void *)session;
2059         mutex_unlock(&pservice->lock);
2060
2061         pr_debug("dev opened\n");
2062         vpu_debug_leave();
2063         return nonseekable_open(inode, filp);
2064 }
2065
2066 static int vpu_service_release(struct inode *inode, struct file *filp)
2067 {
2068         struct vpu_subdev_data *data = container_of(
2069                         inode->i_cdev, struct vpu_subdev_data, cdev);
2070         struct vpu_service_info *pservice = data->pservice;
2071         int task_running;
2072         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2073
2074         vpu_debug_enter();
2075         if (NULL == session)
2076                 return -EINVAL;
2077
2078         task_running = atomic_read(&session->task_running);
2079         if (task_running) {
2080                 pr_err("error: session %d still has %d task running when closing\n",
2081                        session->pid, task_running);
2082                 msleep(50);
2083         }
2084         wake_up(&session->wait);
2085
2086         mutex_lock(&pservice->lock);
2087         /* remove this filp from the asynchronusly notified filp's */
2088         list_del_init(&session->list_session);
2089         vpu_service_session_clear(data, session);
2090         kfree(session);
2091         filp->private_data = NULL;
2092         mutex_unlock(&pservice->lock);
2093
2094         pr_debug("dev closed\n");
2095         vpu_debug_leave();
2096         return 0;
2097 }
2098
2099 static const struct file_operations vpu_service_fops = {
2100         .unlocked_ioctl = vpu_service_ioctl,
2101         .open           = vpu_service_open,
2102         .release        = vpu_service_release,
2103 #ifdef CONFIG_COMPAT
2104         .compat_ioctl   = compat_vpu_service_ioctl,
2105 #endif
2106 };
2107
2108 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2109 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2110 static irqreturn_t vepu_irq(int irq, void *dev_id);
2111 static irqreturn_t vepu_isr(int irq, void *dev_id);
2112 static void get_hw_info(struct vpu_subdev_data *data);
2113
2114 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2115 {
2116         struct device_node *dn = NULL;
2117         struct platform_device *pd = NULL;
2118         struct device *ret = NULL;
2119
2120         dn = of_find_compatible_node(NULL, NULL, compt);
2121         if (!dn) {
2122                 pr_err("can't find device node %s \r\n", compt);
2123                 return NULL;
2124         }
2125
2126         pd = of_find_device_by_node(dn);
2127         if (!pd) {
2128                 pr_err("can't find platform device in device node %s\n", compt);
2129                 return  NULL;
2130         }
2131         ret = &pd->dev;
2132
2133         return ret;
2134 }
2135
2136 #ifdef CONFIG_IOMMU_API
2137 static inline void platform_set_sysmmu(struct device *iommu,
2138                                        struct device *dev)
2139 {
2140         dev->archdata.iommu = iommu;
2141 }
2142 #else
2143 static inline void platform_set_sysmmu(struct device *iommu,
2144                                        struct device *dev)
2145 {
2146 }
2147 #endif
2148
2149 int vcodec_sysmmu_fault_hdl(struct device *dev,
2150                             enum rk_iommu_inttype itype,
2151                             unsigned long pgtable_base,
2152                             unsigned long fault_addr, unsigned int status)
2153 {
2154         struct platform_device *pdev;
2155         struct vpu_service_info *pservice;
2156         struct vpu_subdev_data *data;
2157
2158         vpu_debug_enter();
2159
2160         if (dev == NULL) {
2161                 pr_err("invalid NULL dev\n");
2162                 return 0;
2163         }
2164
2165         pdev = container_of(dev, struct platform_device, dev);
2166         if (pdev == NULL) {
2167                 pr_err("invalid NULL platform_device\n");
2168                 return 0;
2169         }
2170
2171         data = platform_get_drvdata(pdev);
2172         if (data == NULL) {
2173                 pr_err("invalid NULL vpu_subdev_data\n");
2174                 return 0;
2175         }
2176
2177         pservice = data->pservice;
2178         if (pservice == NULL) {
2179                 pr_err("invalid NULL vpu_service_info\n");
2180                 return 0;
2181         }
2182
2183         if (pservice->reg_codec) {
2184                 struct vpu_reg *reg = pservice->reg_codec;
2185                 struct vcodec_mem_region *mem, *n;
2186                 int i = 0;
2187
2188                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2189                 if (!list_empty(&reg->mem_region_list)) {
2190                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2191                                                  reg_lnk) {
2192                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2193                                        mem->reg_idx, i, mem->iova, mem->len);
2194                                 i++;
2195                         }
2196                 } else {
2197                         pr_err("no memory region mapped\n");
2198                 }
2199
2200                 if (reg->data) {
2201                         struct vpu_subdev_data *data = reg->data;
2202                         u32 *base = (u32 *)data->dec_dev.regs;
2203                         u32 len = data->hw_info->dec_reg_num;
2204
2205                         pr_err("current errror register set:\n");
2206
2207                         for (i = 0; i < len; i++)
2208                                 pr_err("reg[%02d] %08x\n",
2209                                        i, readl_relaxed(base + i));
2210                 }
2211
2212                 pr_alert("vcodec, page fault occur, reset hw\n");
2213
2214                 /* reg->reg[101] = 1; */
2215                 vpu_reset(data);
2216         }
2217
2218         return 0;
2219 }
2220
2221 static int vcodec_subdev_probe(struct platform_device *pdev,
2222                                struct vpu_service_info *pservice)
2223 {
2224         int ret = 0;
2225         struct resource *res = NULL;
2226         u32 ioaddr = 0;
2227         u8 *regs = NULL;
2228         struct vpu_hw_info *hw_info = NULL;
2229         struct device *dev = &pdev->dev;
2230         char *name = (char *)dev_name(dev);
2231         struct device_node *np = pdev->dev.of_node;
2232         struct vpu_subdev_data *data =
2233                 devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2234         u32 iommu_en = 0;
2235         char mmu_dev_dts_name[40];
2236
2237         of_property_read_u32(np, "iommu_enabled", &iommu_en);
2238
2239         pr_info("probe device %s\n", dev_name(dev));
2240
2241         data->pservice = pservice;
2242         data->dev = dev;
2243
2244         of_property_read_string(np, "name", (const char **)&name);
2245         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2246
2247         if (pservice->reg_base == 0) {
2248                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2249                 data->regs = devm_ioremap_resource(dev, res);
2250                 if (IS_ERR(data->regs)) {
2251                         ret = PTR_ERR(data->regs);
2252                         goto err;
2253                 }
2254                 ioaddr = res->start;
2255         } else {
2256                 data->regs = pservice->reg_base;
2257                 ioaddr = pservice->ioaddr;
2258         }
2259
2260         clear_bit(MMU_ACTIVATED, &data->state);
2261         vcodec_enter_mode(data);
2262
2263         vpu_service_power_on(pservice);
2264         ret = vpu_service_check_hw(data);
2265         if (ret < 0) {
2266                 vpu_err("error: hw info check faild\n");
2267                 goto err;
2268         }
2269
2270         hw_info = data->hw_info;
2271         regs = (u8 *)data->regs;
2272
2273         if (hw_info->dec_reg_num) {
2274                 data->dec_dev.iosize = hw_info->dec_io_size;
2275                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2276         }
2277
2278         if (hw_info->enc_reg_num) {
2279                 data->enc_dev.iosize = hw_info->enc_io_size;
2280                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2281         }
2282
2283         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2284
2285         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2286         if (data->irq_enc > 0) {
2287                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2288                                                 vepu_irq, vepu_isr,
2289                                                 IRQF_SHARED, dev_name(dev),
2290                                                 (void *)data);
2291                 if (ret) {
2292                         dev_err(dev, "error: can't request vepu irq %d\n",
2293                                 data->irq_enc);
2294                         goto err;
2295                 }
2296         }
2297         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2298         if (data->irq_dec > 0) {
2299                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2300                                                 vdpu_irq, vdpu_isr,
2301                                                 IRQF_SHARED, dev_name(dev),
2302                                                 (void *)data);
2303                 if (ret) {
2304                         dev_err(dev, "error: can't request vdpu irq %d\n",
2305                                 data->irq_dec);
2306                         goto err;
2307                 }
2308         }
2309         atomic_set(&data->dec_dev.irq_count_codec, 0);
2310         atomic_set(&data->dec_dev.irq_count_pp, 0);
2311         atomic_set(&data->enc_dev.irq_count_codec, 0);
2312         atomic_set(&data->enc_dev.irq_count_pp, 0);
2313
2314         if (iommu_en) {
2315                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
2316                         sprintf(mmu_dev_dts_name,
2317                                 HEVC_IOMMU_COMPATIBLE_NAME);
2318                 else if (data->mode == VCODEC_RUNNING_MODE_VPU)
2319                         sprintf(mmu_dev_dts_name,
2320                                 VPU_IOMMU_COMPATIBLE_NAME);
2321                 else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2322                         sprintf(mmu_dev_dts_name, VDEC_IOMMU_COMPATIBLE_NAME);
2323                 else
2324                         sprintf(mmu_dev_dts_name,
2325                                 HEVC_IOMMU_COMPATIBLE_NAME);
2326
2327                 data->mmu_dev =
2328                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2329
2330                 if (data->mmu_dev)
2331                         platform_set_sysmmu(data->mmu_dev, dev);
2332
2333                 rockchip_iovmm_set_fault_handler(dev, vcodec_sysmmu_fault_hdl);
2334         }
2335
2336         get_hw_info(data);
2337         pservice->auto_freq = true;
2338
2339         vcodec_exit_mode(data);
2340         /* create device node */
2341         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2342         if (ret) {
2343                 dev_err(dev, "alloc dev_t failed\n");
2344                 goto err;
2345         }
2346
2347         cdev_init(&data->cdev, &vpu_service_fops);
2348
2349         data->cdev.owner = THIS_MODULE;
2350         data->cdev.ops = &vpu_service_fops;
2351
2352         ret = cdev_add(&data->cdev, data->dev_t, 1);
2353
2354         if (ret) {
2355                 dev_err(dev, "add dev_t failed\n");
2356                 goto err;
2357         }
2358
2359         data->cls = class_create(THIS_MODULE, name);
2360
2361         if (IS_ERR(data->cls)) {
2362                 ret = PTR_ERR(data->cls);
2363                 dev_err(dev, "class_create err:%d\n", ret);
2364                 goto err;
2365         }
2366
2367         data->child_dev = device_create(data->cls, dev,
2368                 data->dev_t, NULL, name);
2369
2370         platform_set_drvdata(pdev, data);
2371
2372         INIT_LIST_HEAD(&data->lnk_service);
2373         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2374
2375 #ifdef CONFIG_DEBUG_FS
2376         data->debugfs_dir = vcodec_debugfs_create_device_dir(name, parent);
2377         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2378                 data->debugfs_file_regs =
2379                         debugfs_create_file("regs", 0664, data->debugfs_dir,
2380                                         data, &debug_vcodec_fops);
2381         else
2382                 vpu_err("create debugfs dir %s failed\n", name);
2383 #endif
2384         return 0;
2385 err:
2386         if (data->child_dev) {
2387                 device_destroy(data->cls, data->dev_t);
2388                 cdev_del(&data->cdev);
2389                 unregister_chrdev_region(data->dev_t, 1);
2390         }
2391
2392         if (data->cls)
2393                 class_destroy(data->cls);
2394         return -1;
2395 }
2396
2397 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2398 {
2399         struct vpu_service_info *pservice = data->pservice;
2400
2401         mutex_lock(&pservice->lock);
2402         cancel_delayed_work_sync(&pservice->power_off_work);
2403         vpu_service_power_off(pservice);
2404         mutex_unlock(&pservice->lock);
2405
2406         device_destroy(data->cls, data->dev_t);
2407         class_destroy(data->cls);
2408         cdev_del(&data->cdev);
2409         unregister_chrdev_region(data->dev_t, 1);
2410
2411 #ifdef CONFIG_DEBUG_FS
2412         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2413                 debugfs_remove_recursive(data->debugfs_dir);
2414 #endif
2415 }
2416
2417 static void vcodec_read_property(struct device_node *np,
2418                                  struct vpu_service_info *pservice)
2419 {
2420         pservice->mode_bit = 0;
2421         pservice->mode_ctrl = 0;
2422         pservice->subcnt = 0;
2423         pservice->grf_base = NULL;
2424
2425         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2426
2427         if (pservice->subcnt > 1) {
2428                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2429                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2430         }
2431 #ifdef CONFIG_MFD_SYSCON
2432         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2433         if (IS_ERR_OR_NULL(pservice->grf)) {
2434                 pservice->grf = NULL;
2435 #ifdef CONFIG_ARM
2436                 pservice->grf_base = RK_GRF_VIRT;
2437 #else
2438                 vpu_err("can't find vpu grf property\n");
2439                 return;
2440 #endif
2441         }
2442 #else
2443 #ifdef CONFIG_ARM
2444         pservice->grf_base = RK_GRF_VIRT;
2445 #else
2446         vpu_err("can't find vpu grf property\n");
2447         return;
2448 #endif
2449 #endif
2450
2451 #ifdef CONFIG_RESET_CONTROLLER
2452         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2453         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2454         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2455
2456         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2457                 pr_warn("No aclk reset resource define\n");
2458                 pservice->rst_a = NULL;
2459         }
2460
2461         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2462                 pr_warn("No hclk reset resource define\n");
2463                 pservice->rst_h = NULL;
2464         }
2465
2466         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2467                 pr_warn("No core reset resource define\n");
2468                 pservice->rst_v = NULL;
2469         }
2470 #endif
2471
2472         of_property_read_string(np, "name", (const char **)&pservice->name);
2473 }
2474
2475 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2476 {
2477         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2478         pservice->curr_mode = -1;
2479
2480         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2481         INIT_LIST_HEAD(&pservice->waiting);
2482         INIT_LIST_HEAD(&pservice->running);
2483         mutex_init(&pservice->lock);
2484         mutex_init(&pservice->shutdown_lock);
2485         atomic_set(&pservice->service_on, 1);
2486
2487         INIT_LIST_HEAD(&pservice->done);
2488         INIT_LIST_HEAD(&pservice->session);
2489         INIT_LIST_HEAD(&pservice->subdev_list);
2490
2491         pservice->reg_pproc     = NULL;
2492         atomic_set(&pservice->total_running, 0);
2493         atomic_set(&pservice->enabled,       0);
2494         atomic_set(&pservice->power_on_cnt,  0);
2495         atomic_set(&pservice->power_off_cnt, 0);
2496         atomic_set(&pservice->reset_request, 0);
2497
2498         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2499         pservice->last.tv64 = 0;
2500
2501         pservice->ion_client = rockchip_ion_client_create("vpu");
2502         if (IS_ERR(pservice->ion_client)) {
2503                 vpu_err("failed to create ion client for vcodec ret %ld\n",
2504                         PTR_ERR(pservice->ion_client));
2505         } else {
2506                 vpu_debug(DEBUG_IOMMU, "vcodec ion client create success!\n");
2507         }
2508 }
2509
2510 static int vcodec_probe(struct platform_device *pdev)
2511 {
2512         int i;
2513         int ret = 0;
2514         struct resource *res = NULL;
2515         struct device *dev = &pdev->dev;
2516         struct device_node *np = pdev->dev.of_node;
2517         struct vpu_service_info *pservice =
2518                 devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
2519
2520         pservice->dev = dev;
2521
2522         vcodec_read_property(np, pservice);
2523         vcodec_init_drvdata(pservice);
2524
2525         if (strncmp(pservice->name, "hevc_service", 12) == 0)
2526                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2527         else if (strncmp(pservice->name, "vpu_service", 11) == 0)
2528                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2529         else if (strncmp(pservice->name, "rkvdec", 6) == 0)
2530                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2531         else
2532                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2533
2534         if (0 > vpu_get_clk(pservice))
2535                 goto err;
2536
2537         if (of_property_read_bool(np, "reg")) {
2538                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2539
2540                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2541                 if (IS_ERR(pservice->reg_base)) {
2542                         vpu_err("ioremap registers base failed\n");
2543                         ret = PTR_ERR(pservice->reg_base);
2544                         goto err;
2545                 }
2546                 pservice->ioaddr = res->start;
2547         } else {
2548                 pservice->reg_base = 0;
2549         }
2550
2551         pm_runtime_enable(dev);
2552
2553         if (of_property_read_bool(np, "subcnt")) {
2554                 for (i = 0; i < pservice->subcnt; i++) {
2555                         struct device_node *sub_np;
2556                         struct platform_device *sub_pdev;
2557
2558                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2559                         sub_pdev = of_find_device_by_node(sub_np);
2560
2561                         vcodec_subdev_probe(sub_pdev, pservice);
2562                 }
2563         } else {
2564                 vcodec_subdev_probe(pdev, pservice);
2565         }
2566
2567         vpu_service_power_off(pservice);
2568
2569         pr_info("init success\n");
2570
2571         return 0;
2572
2573 err:
2574         pr_info("init failed\n");
2575         vpu_service_power_off(pservice);
2576         vpu_put_clk(pservice);
2577         wake_lock_destroy(&pservice->wake_lock);
2578
2579         return ret;
2580 }
2581
2582 static int vcodec_remove(struct platform_device *pdev)
2583 {
2584         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2585
2586         vcodec_subdev_remove(data);
2587
2588         pm_runtime_disable(data->pservice->dev);
2589
2590         return 0;
2591 }
2592
2593 static void vcodec_shutdown(struct platform_device *pdev)
2594 {
2595         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2596         struct vpu_service_info *pservice = data->pservice;
2597
2598         dev_info(&pdev->dev, "%s IN\n", __func__);
2599
2600         mutex_lock(&pservice->shutdown_lock);
2601         atomic_set(&pservice->service_on, 0);
2602         mutex_unlock(&pservice->shutdown_lock);
2603
2604         vcodec_exit_mode(data);
2605
2606         vpu_service_clear(data);
2607         vcodec_subdev_remove(data);
2608
2609         pm_runtime_disable(&pdev->dev);
2610 }
2611
2612 #if defined(CONFIG_OF)
2613 static const struct of_device_id vcodec_service_dt_ids[] = {
2614         {.compatible = "rockchip,vpu_service",},
2615         {.compatible = "rockchip,hevc_service",},
2616         {.compatible = "rockchip,vpu_combo",},
2617         {.compatible = "rockchip,rkvdec",},
2618         {},
2619 };
2620 #endif
2621
2622 static struct platform_driver vcodec_driver = {
2623         .probe = vcodec_probe,
2624         .remove = vcodec_remove,
2625         .shutdown = vcodec_shutdown,
2626         .driver = {
2627                 .name = "vcodec",
2628                 .owner = THIS_MODULE,
2629 #if defined(CONFIG_OF)
2630                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2631 #endif
2632         },
2633 };
2634
2635 static void get_hw_info(struct vpu_subdev_data *data)
2636 {
2637         struct vpu_service_info *pservice = data->pservice;
2638         struct vpu_dec_config *dec = &pservice->dec_config;
2639         struct vpu_enc_config *enc = &pservice->enc_config;
2640
2641         if (cpu_is_rk2928() || cpu_is_rk3036() ||
2642             cpu_is_rk30xx() || cpu_is_rk312x() ||
2643             cpu_is_rk3188())
2644                 dec->max_dec_pic_width = 1920;
2645         else
2646                 dec->max_dec_pic_width = 4096;
2647
2648         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2649                 dec->h264_support = 3;
2650                 dec->jpeg_support = 1;
2651                 dec->mpeg4_support = 2;
2652                 dec->vc1_support = 3;
2653                 dec->mpeg2_support = 1;
2654                 dec->pp_support = 1;
2655                 dec->sorenson_support = 1;
2656                 dec->ref_buf_support = 3;
2657                 dec->vp6_support = 1;
2658                 dec->vp7_support = 1;
2659                 dec->vp8_support = 1;
2660                 dec->avs_support = 1;
2661                 dec->jpeg_ext_support = 0;
2662                 dec->custom_mpeg4_support = 1;
2663                 dec->reserve = 0;
2664                 dec->mvc_support = 1;
2665
2666                 if (!cpu_is_rk3036()) {
2667                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2668
2669                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2670                         enc->h264_enabled = 1;
2671                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2672                         enc->jpeg_enabled = 1;
2673                         enc->vs_enabled = (config_reg >> 24) & 1;
2674                         enc->rgb_enabled = (config_reg >> 28) & 1;
2675                         enc->reg_size = data->reg_size;
2676                         enc->reserv[0] = 0;
2677                         enc->reserv[1] = 0;
2678                 }
2679
2680                 pservice->auto_freq = true;
2681                 vpu_debug(DEBUG_EXTRA_INFO, "vpu_service set to auto frequency mode\n");
2682                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2683
2684                 pservice->bug_dec_addr = cpu_is_rk30xx();
2685         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2686                 pservice->auto_freq = true;
2687                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2688         } else {
2689                 /* disable frequency switch in hevc.*/
2690                 pservice->auto_freq = false;
2691         }
2692 }
2693
2694 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2695 {
2696         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2697                   task->name, irq_status, task->error_mask);
2698
2699         return (task->error_mask & irq_status) ? true : false;
2700 }
2701
2702 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2703 {
2704         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2705         struct vpu_service_info *pservice = data->pservice;
2706         struct vpu_task_info *task = NULL;
2707         struct vpu_device *dev = &data->dec_dev;
2708         u32 hw_id = data->hw_info->hw_id;
2709         u32 raw_status;
2710         u32 dec_status;
2711
2712         task = &data->task_info[TASK_DEC];
2713
2714         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2715         dec_status = raw_status;
2716
2717         vpu_debug(DEBUG_TASK_INFO, "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2718                   task->reg_irq, dec_status,
2719                   task->irq_mask, task->ready_mask, task->error_mask);
2720
2721         if (dec_status & task->irq_mask) {
2722                 time_record(task, 1);
2723                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2724                           dec_status);
2725                 if ((dec_status & 0x40001) == 0x40001) {
2726                         do {
2727                                 dec_status =
2728                                         readl_relaxed(dev->regs +
2729                                                 task->reg_irq);
2730                         } while ((dec_status & 0x40001) == 0x40001);
2731                 }
2732
2733                 if (check_irq_err(task, dec_status))
2734                         atomic_add(1, &pservice->reset_request);
2735
2736                 writel_relaxed(0, dev->regs + task->reg_irq);
2737
2738                 /*
2739                  * NOTE: rkvdec need to reset after each task to avoid timeout
2740                  *       error on H.264 switch to H.265
2741                  */
2742                 if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2743                         writel(0x100000, dev->regs + task->reg_irq);
2744
2745                 /* set clock gating to save power */
2746                 writel(task->gating_mask, dev->regs + task->reg_en);
2747
2748                 atomic_add(1, &dev->irq_count_codec);
2749                 time_diff(task);
2750         }
2751
2752         task = &data->task_info[TASK_PP];
2753         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2754                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2755
2756                 if (pp_status & task->irq_mask) {
2757                         time_record(task, 1);
2758                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2759                                   pp_status);
2760
2761                         if (check_irq_err(task, dec_status))
2762                                 atomic_add(1, &pservice->reset_request);
2763
2764                         /* clear pp IRQ */
2765                         writel_relaxed(pp_status & (~task->reg_irq),
2766                                        dev->regs + task->irq_mask);
2767                         atomic_add(1, &dev->irq_count_pp);
2768                         time_diff(task);
2769                 }
2770         }
2771
2772         pservice->irq_status = raw_status;
2773
2774         if (atomic_read(&dev->irq_count_pp) ||
2775             atomic_read(&dev->irq_count_codec))
2776                 return IRQ_WAKE_THREAD;
2777         else
2778                 return IRQ_NONE;
2779 }
2780
2781 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2782 {
2783         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2784         struct vpu_service_info *pservice = data->pservice;
2785         struct vpu_device *dev = &data->dec_dev;
2786
2787         mutex_lock(&pservice->lock);
2788         if (atomic_read(&dev->irq_count_codec)) {
2789                 atomic_sub(1, &dev->irq_count_codec);
2790                 if (pservice->reg_codec == NULL) {
2791                         vpu_err("error: dec isr with no task waiting\n");
2792                 } else {
2793                         reg_from_run_to_done(data, pservice->reg_codec);
2794                         /* avoid vpu timeout and can't recover problem */
2795                         VDPU_SOFT_RESET(data->regs);
2796                 }
2797         }
2798
2799         if (atomic_read(&dev->irq_count_pp)) {
2800                 atomic_sub(1, &dev->irq_count_pp);
2801                 if (pservice->reg_pproc == NULL)
2802                         vpu_err("error: pp isr with no task waiting\n");
2803                 else
2804                         reg_from_run_to_done(data, pservice->reg_pproc);
2805         }
2806         try_set_reg(data);
2807         mutex_unlock(&pservice->lock);
2808         return IRQ_HANDLED;
2809 }
2810
2811 static irqreturn_t vepu_irq(int irq, void *dev_id)
2812 {
2813         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2814         struct vpu_service_info *pservice = data->pservice;
2815         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2816         struct vpu_device *dev = &data->enc_dev;
2817         u32 irq_status;
2818
2819         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2820
2821         vpu_debug(DEBUG_TASK_INFO, "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2822                   task->reg_irq, irq_status,
2823                   task->irq_mask, task->ready_mask, task->error_mask);
2824
2825         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2826
2827         if (likely(irq_status & task->irq_mask)) {
2828                 time_record(task, 1);
2829
2830                 if (check_irq_err(task, irq_status))
2831                         atomic_add(1, &pservice->reset_request);
2832
2833                 /* clear enc IRQ */
2834                 writel_relaxed(irq_status & (~task->irq_mask),
2835                                dev->regs + task->reg_irq);
2836
2837                 atomic_add(1, &dev->irq_count_codec);
2838                 time_diff(task);
2839         }
2840
2841         pservice->irq_status = irq_status;
2842
2843         if (atomic_read(&dev->irq_count_codec))
2844                 return IRQ_WAKE_THREAD;
2845         else
2846                 return IRQ_NONE;
2847 }
2848
2849 static irqreturn_t vepu_isr(int irq, void *dev_id)
2850 {
2851         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2852         struct vpu_service_info *pservice = data->pservice;
2853         struct vpu_device *dev = &data->enc_dev;
2854
2855         mutex_lock(&pservice->lock);
2856         if (atomic_read(&dev->irq_count_codec)) {
2857                 atomic_sub(1, &dev->irq_count_codec);
2858                 if (NULL == pservice->reg_codec)
2859                         vpu_err("error: enc isr with no task waiting\n");
2860                 else
2861                         reg_from_run_to_done(data, pservice->reg_codec);
2862         }
2863         try_set_reg(data);
2864         mutex_unlock(&pservice->lock);
2865         return IRQ_HANDLED;
2866 }
2867
2868 static int __init vcodec_service_init(void)
2869 {
2870         int ret = platform_driver_register(&vcodec_driver);
2871
2872         if (ret) {
2873                 vpu_err("Platform device register failed (%d).\n", ret);
2874                 return ret;
2875         }
2876
2877 #ifdef CONFIG_DEBUG_FS
2878         vcodec_debugfs_init();
2879 #endif
2880
2881         return ret;
2882 }
2883
2884 static void __exit vcodec_service_exit(void)
2885 {
2886 #ifdef CONFIG_DEBUG_FS
2887         vcodec_debugfs_exit();
2888 #endif
2889
2890         platform_driver_unregister(&vcodec_driver);
2891 }
2892
2893 module_init(vcodec_service_init);
2894 module_exit(vcodec_service_exit);
2895 MODULE_LICENSE("Proprietary");
2896
2897 #ifdef CONFIG_DEBUG_FS
2898 #include <linux/seq_file.h>
2899
2900 static int vcodec_debugfs_init(void)
2901 {
2902         parent = debugfs_create_dir("vcodec", NULL);
2903         if (!parent)
2904                 return -1;
2905
2906         return 0;
2907 }
2908
2909 static void vcodec_debugfs_exit(void)
2910 {
2911         debugfs_remove(parent);
2912 }
2913
2914 static struct dentry *vcodec_debugfs_create_device_dir(
2915                 char *dirname, struct dentry *parent)
2916 {
2917         return debugfs_create_dir(dirname, parent);
2918 }
2919
2920 static int debug_vcodec_show(struct seq_file *s, void *unused)
2921 {
2922         struct vpu_subdev_data *data = s->private;
2923         struct vpu_service_info *pservice = data->pservice;
2924         unsigned int i, n;
2925         struct vpu_reg *reg, *reg_tmp;
2926         struct vpu_session *session, *session_tmp;
2927
2928         mutex_lock(&pservice->lock);
2929         vpu_service_power_on(pservice);
2930         if (data->hw_info->hw_id != HEVC_ID) {
2931                 seq_puts(s, "\nENC Registers:\n");
2932                 n = data->enc_dev.iosize >> 2;
2933
2934                 for (i = 0; i < n; i++)
2935                         seq_printf(s, "\tswreg%d = %08X\n", i,
2936                                    readl_relaxed(data->enc_dev.regs + i));
2937         }
2938
2939         seq_puts(s, "\nDEC Registers:\n");
2940
2941         n = data->dec_dev.iosize >> 2;
2942         for (i = 0; i < n; i++)
2943                 seq_printf(s, "\tswreg%d = %08X\n", i,
2944                            readl_relaxed(data->dec_dev.regs + i));
2945
2946         seq_puts(s, "\nvpu service status:\n");
2947
2948         list_for_each_entry_safe(session, session_tmp,
2949                                  &pservice->session, list_session) {
2950                 seq_printf(s, "session pid %d type %d:\n",
2951                            session->pid, session->type);
2952
2953                 list_for_each_entry_safe(reg, reg_tmp,
2954                                          &session->waiting, session_link) {
2955                         seq_printf(s, "waiting register set %p\n", reg);
2956                 }
2957                 list_for_each_entry_safe(reg, reg_tmp,
2958                                          &session->running, session_link) {
2959                         seq_printf(s, "running register set %p\n", reg);
2960                 }
2961                 list_for_each_entry_safe(reg, reg_tmp,
2962                                          &session->done, session_link) {
2963                         seq_printf(s, "done    register set %p\n", reg);
2964                 }
2965         }
2966
2967         seq_printf(s, "\npower counter: on %d off %d\n",
2968                    atomic_read(&pservice->power_on_cnt),
2969                    atomic_read(&pservice->power_off_cnt));
2970
2971         mutex_unlock(&pservice->lock);
2972         vpu_service_power_off(pservice);
2973
2974         return 0;
2975 }
2976
2977 static int debug_vcodec_open(struct inode *inode, struct file *file)
2978 {
2979         return single_open(file, debug_vcodec_show, inode->i_private);
2980 }
2981
2982 #endif
2983