drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/clk.h>
  20 #include <linux/compat.h>
  21 #include <linux/delay.h>
  22 #include <linux/init.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/module.h>
  25 #include <linux/fs.h>
  26 #include <linux/mm.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/reset.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31 #include <linux/wakelock.h>
  32 #include <linux/cdev.h>
  33 #include <linux/of.h>
  34 #include <linux/of_platform.h>
  35 #include <linux/of_irq.h>
  36 #include <linux/regmap.h>
  37 #include <linux/mfd/syscon.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/debugfs.h>
  40 #include <linux/pm_runtime.h>
  41
  42 #include <linux/rockchip/cpu.h>
  43 #include <linux/rockchip/cru.h>
  44 #include <linux/rockchip/pmu.h>
  45 #include <linux/rockchip/grf.h>
  46
  47 #if defined(CONFIG_ION_ROCKCHIP)
  48 #include <linux/rockchip_ion.h>
  49 #endif
  50
  51 #include <linux/rockchip-iovmm.h>
  52 #include <linux/dma-buf.h>
  53
  54 #include "vcodec_hw_info.h"
  55 #include "vcodec_hw_vpu.h"
  56 #include "vcodec_hw_rkv.h"
  57 #include "vcodec_hw_vpu2.h"
  58
  59 #include "vcodec_service.h"
  60
  61 /*
  62  * debug flag usage:
  63  * +------+-------------------+
  64  * | 8bit |      24bit        |
  65  * +------+-------------------+
  66  *  0~23 bit is for different information type
  67  * 24~31 bit is for information print format
  68  */
  69
  70 #define DEBUG_POWER                             0x00000001
  71 #define DEBUG_CLOCK                             0x00000002
  72 #define DEBUG_IRQ_STATUS                        0x00000004
  73 #define DEBUG_IOMMU                             0x00000008
  74 #define DEBUG_IOCTL                             0x00000010
  75 #define DEBUG_FUNCTION                          0x00000020
  76 #define DEBUG_REGISTER                          0x00000040
  77 #define DEBUG_EXTRA_INFO                        0x00000080
  78 #define DEBUG_TIMING                            0x00000100
  79 #define DEBUG_TASK_INFO                         0x00000200
  80
  81 #define DEBUG_SET_REG                           0x00001000
  82 #define DEBUG_GET_REG                           0x00002000
  83 #define DEBUG_PPS_FILL                          0x00004000
  84 #define DEBUG_IRQ_CHECK                         0x00008000
  85 #define DEBUG_CACHE_32B                         0x00010000
  86
  87 #define PRINT_FUNCTION                          0x80000000
  88 #define PRINT_LINE                              0x40000000
  89
  90 static int debug;
  91 module_param(debug, int, S_IRUGO | S_IWUSR);
  92 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  93
  94 #define VCODEC_CLOCK_ENABLE     1
  95
  96 /*
  97  * hardware information organization
  98  *
  99  * In order to support multiple hardware with different version the hardware
 100  * information is organized as follow:
 101  *
 102  * 1. First, index hardware by register size / position.
 103  *    These information is fix for each hardware and do not relate to runtime
 104  *    work flow. It only related to resource allocation.
 105  *    Descriptor: struct vpu_hw_info
 106  *
 107  * 2. Then, index hardware by runtime configuration
 108  *    These information is related to runtime setting behave including enable
 109  *    register, irq register and other key control flag
 110  *    Descriptor: struct vpu_task_info
 111  *
 112  * 3. Final, on iommu case the fd translation is required
 113  *    Descriptor: struct vpu_trans_info
 114  */
 115
 116 enum VPU_FREQ {
 117         VPU_FREQ_200M,
 118         VPU_FREQ_266M,
 119         VPU_FREQ_300M,
 120         VPU_FREQ_400M,
 121         VPU_FREQ_500M,
 122         VPU_FREQ_600M,
 123         VPU_FREQ_DEFAULT,
 124         VPU_FREQ_BUT,
 125 };
 126
 127 struct extra_info_elem {
 128         u32 index;
 129         u32 offset;
 130 };
 131
 132 #define EXTRA_INFO_MAGIC        0x4C4A46
 133
 134 struct extra_info_for_iommu {
 135         u32 magic;
 136         u32 cnt;
 137         struct extra_info_elem elem[20];
 138 };
 139
 140 #define MHZ                                     (1000*1000)
 141 #define SIZE_REG(reg)                           ((reg)*4)
 142
 143 static struct vcodec_info vcodec_info_set[] = {
 144         [0] = {
 145                 .hw_id          = VPU_ID_8270,
 146                 .hw_info        = &hw_vpu_8270,
 147                 .task_info      = task_vpu,
 148                 .trans_info     = trans_vpu,
 149         },
 150         [1] = {
 151                 .hw_id          = VPU_ID_4831,
 152                 .hw_info        = &hw_vpu_4831,
 153                 .task_info      = task_vpu,
 154                 .trans_info     = trans_vpu,
 155         },
 156         [2] = {
 157                 .hw_id          = VPU_DEC_ID_9190,
 158                 .hw_info        = &hw_vpu_9190,
 159                 .task_info      = task_vpu,
 160                 .trans_info     = trans_vpu,
 161         },
 162         [3] = {
 163                 .hw_id          = HEVC_ID,
 164                 .hw_info        = &hw_rkhevc,
 165                 .task_info      = task_rkv,
 166                 .trans_info     = trans_rkv,
 167         },
 168         [4] = {
 169                 .hw_id          = RKV_DEC_ID,
 170                 .hw_info        = &hw_rkvdec,
 171                 .task_info      = task_rkv,
 172                 .trans_info     = trans_rkv,
 173         },
 174         [5] = {
 175                 .hw_id          = VPU2_ID,
 176                 .hw_info        = &hw_vpu2,
 177                 .task_info      = task_vpu2,
 178                 .trans_info     = trans_vpu2,
 179         },
 180 };
 181
 182 #define DEBUG
 183 #ifdef DEBUG
 184 #define vpu_debug_func(type, fmt, args...)                      \
 185         do {                                                    \
 186                 if (unlikely(debug & type)) {                   \
 187                         pr_info("%s:%d: " fmt,                  \
 188                                  __func__, __LINE__, ##args);   \
 189                 }                                               \
 190         } while (0)
 191 #define vpu_debug(type, fmt, args...)                           \
 192         do {                                                    \
 193                 if (unlikely(debug & type)) {                   \
 194                         pr_info(fmt, ##args);                   \
 195                 }                                               \
 196         } while (0)
 197 #else
 198 #define vpu_debug_func(level, fmt, args...)
 199 #define vpu_debug(level, fmt, args...)
 200 #endif
 201
 202 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 203 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 204
 205 #define vpu_err(fmt, args...)                           \
 206                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 207
 208 enum VPU_DEC_FMT {
 209         VPU_DEC_FMT_H264,
 210         VPU_DEC_FMT_MPEG4,
 211         VPU_DEC_FMT_H263,
 212         VPU_DEC_FMT_JPEG,
 213         VPU_DEC_FMT_VC1,
 214         VPU_DEC_FMT_MPEG2,
 215         VPU_DEC_FMT_MPEG1,
 216         VPU_DEC_FMT_VP6,
 217         VPU_DEC_FMT_RESERV0,
 218         VPU_DEC_FMT_VP7,
 219         VPU_DEC_FMT_VP8,
 220         VPU_DEC_FMT_AVS,
 221         VPU_DEC_FMT_RES
 222 };
 223
 224 /**
 225  * struct for process session which connect to vpu
 226  *
 227  * @author ChenHengming (2011-5-3)
 228  */
 229 struct vpu_session {
 230         enum VPU_CLIENT_TYPE type;
 231         /* a linked list of data so we can access them for debugging */
 232         struct list_head list_session;
 233         /* a linked list of register data waiting for process */
 234         struct list_head waiting;
 235         /* a linked list of register data in processing */
 236         struct list_head running;
 237         /* a linked list of register data processed */
 238         struct list_head done;
 239         wait_queue_head_t wait;
 240         pid_t pid;
 241         atomic_t task_running;
 242 };
 243
 244 /**
 245  * struct for process register set
 246  *
 247  * @author ChenHengming (2011-5-4)
 248  */
 249 struct vpu_reg {
 250         enum VPU_CLIENT_TYPE type;
 251         enum VPU_FREQ freq;
 252         struct vpu_session *session;
 253         struct vpu_subdev_data *data;
 254         struct vpu_task_info *task;
 255         const struct vpu_trans_info *trans;
 256
 257         /* link to vpu service session */
 258         struct list_head session_link;
 259         /* link to register set list */
 260         struct list_head status_link;
 261
 262         unsigned long size;
 263         struct list_head mem_region_list;
 264         u32 dec_base;
 265         u32 *reg;
 266 };
 267
 268 struct vpu_device {
 269         atomic_t irq_count_codec;
 270         atomic_t irq_count_pp;
 271         unsigned int iosize;
 272         u32 *regs;
 273 };
 274
 275 enum vcodec_device_id {
 276         VCODEC_DEVICE_ID_VPU,
 277         VCODEC_DEVICE_ID_HEVC,
 278         VCODEC_DEVICE_ID_COMBO,
 279         VCODEC_DEVICE_ID_RKVDEC,
 280         VCODEC_DEVICE_ID_BUTT
 281 };
 282
 283 enum VCODEC_RUNNING_MODE {
 284         VCODEC_RUNNING_MODE_NONE = -1,
 285         VCODEC_RUNNING_MODE_VPU,
 286         VCODEC_RUNNING_MODE_HEVC,
 287         VCODEC_RUNNING_MODE_RKVDEC
 288 };
 289
 290 struct vcodec_mem_region {
 291         struct list_head srv_lnk;
 292         struct list_head reg_lnk;
 293         struct list_head session_lnk;
 294         unsigned long iova;     /* virtual address for iommu */
 295         unsigned long len;
 296         u32 reg_idx;
 297         struct ion_handle *hdl;
 298 };
 299
 300 enum vpu_ctx_state {
 301         MMU_ACTIVATED   = BIT(0)
 302 };
 303
 304 struct vpu_subdev_data {
 305         struct cdev cdev;
 306         dev_t dev_t;
 307         struct class *cls;
 308         struct device *child_dev;
 309
 310         int irq_enc;
 311         int irq_dec;
 312         struct vpu_service_info *pservice;
 313
 314         u32 *regs;
 315         enum VCODEC_RUNNING_MODE mode;
 316         struct list_head lnk_service;
 317
 318         struct device *dev;
 319
 320         struct vpu_device enc_dev;
 321         struct vpu_device dec_dev;
 322
 323         enum VPU_HW_ID hw_id;
 324         struct vpu_hw_info *hw_info;
 325         struct vpu_task_info *task_info;
 326         const struct vpu_trans_info *trans_info;
 327
 328         u32 reg_size;
 329         unsigned long state;
 330
 331 #ifdef CONFIG_DEBUG_FS
 332         struct dentry *debugfs_dir;
 333         struct dentry *debugfs_file_regs;
 334 #endif
 335
 336         struct device *mmu_dev;
 337 };
 338
 339 struct vpu_service_info {
 340         struct wake_lock wake_lock;
 341         struct delayed_work power_off_work;
 342         ktime_t last; /* record previous power-on time */
 343         /* vpu service structure global lock */
 344         struct mutex lock;
 345         /* link to link_reg in struct vpu_reg */
 346         struct list_head waiting;
 347         /* link to link_reg in struct vpu_reg */
 348         struct list_head running;
 349         /* link to link_reg in struct vpu_reg */
 350         struct list_head done;
 351         /* link to list_session in struct vpu_session */
 352         struct list_head session;
 353         atomic_t total_running;
 354         atomic_t enabled;
 355         atomic_t power_on_cnt;
 356         atomic_t power_off_cnt;
 357         atomic_t service_on;
 358         struct mutex shutdown_lock;
 359         struct vpu_reg *reg_codec;
 360         struct vpu_reg *reg_pproc;
 361         struct vpu_reg *reg_resev;
 362         struct vpu_dec_config dec_config;
 363         struct vpu_enc_config enc_config;
 364
 365         bool auto_freq;
 366         bool bug_dec_addr;
 367         atomic_t freq_status;
 368
 369         struct clk *aclk_vcodec;
 370         struct clk *hclk_vcodec;
 371         struct clk *clk_core;
 372         struct clk *clk_cabac;
 373         struct clk *pd_video;
 374
 375 #ifdef CONFIG_RESET_CONTROLLER
 376         struct reset_control *rst_a;
 377         struct reset_control *rst_h;
 378         struct reset_control *rst_v;
 379 #endif
 380         struct device *dev;
 381
 382         u32 irq_status;
 383         atomic_t reset_request;
 384         struct ion_client *ion_client;
 385         struct list_head mem_region_list;
 386
 387         enum vcodec_device_id dev_id;
 388
 389         enum VCODEC_RUNNING_MODE curr_mode;
 390         u32 prev_mode;
 391
 392         struct delayed_work simulate_work;
 393
 394         u32 mode_bit;
 395         u32 mode_ctrl;
 396         u32 *reg_base;
 397         u32 ioaddr;
 398         struct regmap *grf;
 399         u32 *grf_base;
 400
 401         char *name;
 402
 403         u32 subcnt;
 404         struct list_head subdev_list;
 405 };
 406
 407 struct vpu_request {
 408         u32 *req;
 409         u32 size;
 410 };
 411
 412 #ifdef CONFIG_COMPAT
 413 struct compat_vpu_request {
 414         compat_uptr_t req;
 415         u32 size;
 416 };
 417 #endif
 418
 419 /* debugfs root directory for all device (vpu, hevc).*/
 420 static struct dentry *parent;
 421
 422 #ifdef CONFIG_DEBUG_FS
 423 static int vcodec_debugfs_init(void);
 424 static void vcodec_debugfs_exit(void);
 425 static struct dentry *vcodec_debugfs_create_device_dir(
 426                 char *dirname, struct dentry *parent);
 427 static int debug_vcodec_open(struct inode *inode, struct file *file);
 428
 429 static const struct file_operations debug_vcodec_fops = {
 430         .open = debug_vcodec_open,
 431         .read = seq_read,
 432         .llseek = seq_lseek,
 433         .release = single_release,
 434 };
 435 #endif
 436
 437 #define VDPU_SOFT_RESET_REG     101
 438 #define VDPU_CLEAN_CACHE_REG    516
 439 #define VEPU_CLEAN_CACHE_REG    772
 440 #define HEVC_CLEAN_CACHE_REG    260
 441
 442 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 443
 444 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 445 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 446 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 447 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 448
 449 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 450 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 451
 452 static void time_record(struct vpu_task_info *task, int is_end)
 453 {
 454         if (unlikely(debug & DEBUG_TIMING) && task)
 455                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 456 }
 457
 458 static void time_diff(struct vpu_task_info *task)
 459 {
 460         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 461                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 462                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 463 }
 464
 465 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 466 {
 467         int bits;
 468         u32 raw = 0;
 469         struct vpu_service_info *pservice = data->pservice;
 470         struct vpu_subdev_data *subdata, *n;
 471
 472         if (pservice->subcnt < 2) {
 473                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 474                         set_bit(MMU_ACTIVATED, &data->state);
 475                         if (atomic_read(&pservice->enabled))
 476                                 rockchip_iovmm_activate(data->dev);
 477                         else
 478                                 BUG_ON(!atomic_read(&pservice->enabled));
 479                 }
 480                 return;
 481         }
 482
 483         if (pservice->curr_mode == data->mode)
 484                 return;
 485
 486         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 487         list_for_each_entry_safe(subdata, n,
 488                                  &pservice->subdev_list, lnk_service) {
 489                 if (data != subdata && subdata->mmu_dev &&
 490                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 491                         clear_bit(MMU_ACTIVATED, &subdata->state);
 492                         rockchip_iovmm_deactivate(subdata->dev);
 493                 }
 494         }
 495         bits = 1 << pservice->mode_bit;
 496 #ifdef CONFIG_MFD_SYSCON
 497         if (pservice->grf) {
 498                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 499
 500                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 501                         regmap_write(pservice->grf, pservice->mode_ctrl,
 502                                      raw | bits | (bits << 16));
 503                 else
 504                         regmap_write(pservice->grf, pservice->mode_ctrl,
 505                                      (raw & (~bits)) | (bits << 16));
 506         } else if (pservice->grf_base) {
 507                 u32 *grf_base = pservice->grf_base;
 508
 509                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 510                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 511                         writel_relaxed(raw | bits | (bits << 16),
 512                                        grf_base + pservice->mode_ctrl / 4);
 513                 else
 514                         writel_relaxed((raw & (~bits)) | (bits << 16),
 515                                        grf_base + pservice->mode_ctrl / 4);
 516         } else {
 517                 vpu_err("no grf resource define, switch decoder failed\n");
 518                 return;
 519         }
 520 #else
 521         if (pservice->grf_base) {
 522                 u32 *grf_base = pservice->grf_base;
 523
 524                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 525                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 526                         writel_relaxed(raw | bits | (bits << 16),
 527                                        grf_base + pservice->mode_ctrl / 4);
 528                 else
 529                         writel_relaxed((raw & (~bits)) | (bits << 16),
 530                                        grf_base + pservice->mode_ctrl / 4);
 531         } else {
 532                 vpu_err("no grf resource define, switch decoder failed\n");
 533                 return;
 534         }
 535 #endif
 536         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 537                 set_bit(MMU_ACTIVATED, &data->state);
 538                 if (atomic_read(&pservice->enabled))
 539                         rockchip_iovmm_activate(data->dev);
 540                 else
 541                         BUG_ON(!atomic_read(&pservice->enabled));
 542         }
 543
 544         pservice->prev_mode = pservice->curr_mode;
 545         pservice->curr_mode = data->mode;
 546 }
 547
 548 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 549 {
 550         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 551                 clear_bit(MMU_ACTIVATED, &data->state);
 552                 rockchip_iovmm_deactivate(data->dev);
 553         }
 554         /*
 555          * In case of VPU Combo, it require HW switch its running mode
 556          * before the other HW component start work. set current HW running
 557          * mode to none, can ensure HW switch to its reqired mode properly.
 558          */
 559         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 560 }
 561
 562 static int vpu_get_clk(struct vpu_service_info *pservice)
 563 {
 564 #if VCODEC_CLOCK_ENABLE
 565         struct device *dev = pservice->dev;
 566
 567         switch (pservice->dev_id) {
 568         case VCODEC_DEVICE_ID_HEVC:
 569                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 570                 if (IS_ERR(pservice->pd_video)) {
 571                         dev_err(dev, "failed on clk_get pd_hevc\n");
 572                         return -1;
 573                 }
 574         case VCODEC_DEVICE_ID_COMBO:
 575         case VCODEC_DEVICE_ID_RKVDEC:
 576                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 577                 if (IS_ERR(pservice->clk_cabac)) {
 578                         dev_err(dev, "failed on clk_get clk_cabac\n");
 579                         pservice->clk_cabac = NULL;
 580                 }
 581                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 582                 if (IS_ERR(pservice->clk_core)) {
 583                         dev_err(dev, "failed on clk_get clk_core\n");
 584                         return -1;
 585                 }
 586         case VCODEC_DEVICE_ID_VPU:
 587                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 588                 if (IS_ERR(pservice->aclk_vcodec)) {
 589                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 590                         return -1;
 591                 }
 592
 593                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 594                 if (IS_ERR(pservice->hclk_vcodec)) {
 595                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 596                         return -1;
 597                 }
 598                 if (pservice->pd_video == NULL) {
 599                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 600                         if (IS_ERR(pservice->pd_video)) {
 601                                 pservice->pd_video = NULL;
 602                                 dev_info(dev, "do not have pd_video\n");
 603                         }
 604                 }
 605                 break;
 606         default:
 607                 break;
 608         }
 609
 610         return 0;
 611 #else
 612         return 0;
 613 #endif
 614 }
 615
 616 static void vpu_put_clk(struct vpu_service_info *pservice)
 617 {
 618 #if VCODEC_CLOCK_ENABLE
 619         if (pservice->pd_video)
 620                 devm_clk_put(pservice->dev, pservice->pd_video);
 621         if (pservice->aclk_vcodec)
 622                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 623         if (pservice->hclk_vcodec)
 624                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 625         if (pservice->clk_core)
 626                 devm_clk_put(pservice->dev, pservice->clk_core);
 627         if (pservice->clk_cabac)
 628                 devm_clk_put(pservice->dev, pservice->clk_cabac);
 629 #endif
 630 }
 631
 632 static void vpu_reset(struct vpu_subdev_data *data)
 633 {
 634         struct vpu_service_info *pservice = data->pservice;
 635         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 636
 637         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 638                 type = IDLE_REQ_HEVC;
 639
 640         pr_info("%s: resetting...", dev_name(pservice->dev));
 641
 642 #if defined(CONFIG_ARCH_RK29)
 643         clk_disable(aclk_ddr_vepu);
 644         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 645         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 646         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 647         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 648         mdelay(10);
 649         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 650         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 651         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 652         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 653         clk_enable(aclk_ddr_vepu);
 654 #elif defined(CONFIG_ARCH_RK30)
 655         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 656         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 657         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 658         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 659         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 660         mdelay(1);
 661         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 662         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 663         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 664         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 665         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 666 #else
 667 #endif
 668         WARN_ON(pservice->reg_codec != NULL);
 669         WARN_ON(pservice->reg_pproc != NULL);
 670         WARN_ON(pservice->reg_resev != NULL);
 671         pservice->reg_codec = NULL;
 672         pservice->reg_pproc = NULL;
 673         pservice->reg_resev = NULL;
 674
 675         pr_info("for 3288/3368...");
 676 #ifdef CONFIG_RESET_CONTROLLER
 677         if (pservice->rst_a && pservice->rst_h) {
 678                 pr_info("reset in\n");
 679                 if (pservice->rst_v)
 680                         reset_control_assert(pservice->rst_v);
 681                 reset_control_assert(pservice->rst_a);
 682                 reset_control_assert(pservice->rst_h);
 683                 udelay(5);
 684                 reset_control_deassert(pservice->rst_h);
 685                 reset_control_deassert(pservice->rst_a);
 686                 if (pservice->rst_v)
 687                         reset_control_deassert(pservice->rst_v);
 688         }
 689 #endif
 690
 691         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 692                 clear_bit(MMU_ACTIVATED, &data->state);
 693                 if (atomic_read(&pservice->enabled))
 694                         rockchip_iovmm_deactivate(data->dev);
 695                 else
 696                         BUG_ON(!atomic_read(&pservice->enabled));
 697         }
 698
 699         atomic_set(&pservice->reset_request, 0);
 700         pr_info("done\n");
 701 }
 702
 703 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 704 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 705                                       struct vpu_session *session)
 706 {
 707         struct vpu_reg *reg, *n;
 708
 709         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 710                 reg_deinit(data, reg);
 711         }
 712         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 713                 reg_deinit(data, reg);
 714         }
 715         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 716                 reg_deinit(data, reg);
 717         }
 718 }
 719
 720 static void vpu_service_clear(struct vpu_subdev_data *data)
 721 {
 722         struct vpu_reg *reg, *n;
 723         struct vpu_session *session, *s;
 724         struct vpu_service_info *pservice = data->pservice;
 725
 726         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 727                 reg_deinit(data, reg);
 728         }
 729
 730         /* wake up session wait event to prevent the timeout hw reset
 731          * during reboot procedure.
 732          */
 733         list_for_each_entry_safe(session, s,
 734                                  &pservice->session, list_session)
 735                 wake_up(&session->wait);
 736 }
 737
 738 static void vpu_service_dump(struct vpu_service_info *pservice)
 739 {
 740 }
 741
 742
 743 static void vpu_service_power_off(struct vpu_service_info *pservice)
 744 {
 745         int total_running;
 746         struct vpu_subdev_data *data = NULL, *n;
 747         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 748
 749         if (!ret)
 750                 return;
 751
 752         total_running = atomic_read(&pservice->total_running);
 753         if (total_running) {
 754                 pr_alert("alert: power off when %d task running!!\n",
 755                          total_running);
 756                 mdelay(50);
 757                 pr_alert("alert: delay 50 ms for running task\n");
 758                 vpu_service_dump(pservice);
 759         }
 760
 761         pr_info("%s: power off...", dev_name(pservice->dev));
 762
 763         udelay(5);
 764
 765         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 766                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 767                         clear_bit(MMU_ACTIVATED, &data->state);
 768                         rockchip_iovmm_deactivate(data->dev);
 769                 }
 770         }
 771         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 772
 773 #if VCODEC_CLOCK_ENABLE
 774                 if (pservice->pd_video)
 775                         clk_disable_unprepare(pservice->pd_video);
 776                 if (pservice->hclk_vcodec)
 777                         clk_disable_unprepare(pservice->hclk_vcodec);
 778                 if (pservice->aclk_vcodec)
 779                         clk_disable_unprepare(pservice->aclk_vcodec);
 780                 if (pservice->clk_core)
 781                         clk_disable_unprepare(pservice->clk_core);
 782                 if (pservice->clk_cabac)
 783                         clk_disable_unprepare(pservice->clk_cabac);
 784 #endif
 785         pm_runtime_put(pservice->dev);
 786
 787         atomic_add(1, &pservice->power_off_cnt);
 788         wake_unlock(&pservice->wake_lock);
 789         pr_info("done\n");
 790 }
 791
 792 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 793 {
 794         queue_delayed_work(system_wq, &pservice->power_off_work,
 795                            VPU_POWER_OFF_DELAY);
 796 }
 797
 798 static void vpu_power_off_work(struct work_struct *work_s)
 799 {
 800         struct delayed_work *dlwork = container_of(work_s,
 801                         struct delayed_work, work);
 802         struct vpu_service_info *pservice = container_of(dlwork,
 803                         struct vpu_service_info, power_off_work);
 804
 805         if (mutex_trylock(&pservice->lock)) {
 806                 vpu_service_power_off(pservice);
 807                 mutex_unlock(&pservice->lock);
 808         } else {
 809                 /* Come back later if the device is busy... */
 810                 vpu_queue_power_off_work(pservice);
 811         }
 812 }
 813
 814 static void vpu_service_power_on(struct vpu_service_info *pservice)
 815 {
 816         int ret;
 817         ktime_t now = ktime_get();
 818
 819         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC) {
 820                 cancel_delayed_work_sync(&pservice->power_off_work);
 821                 vpu_queue_power_off_work(pservice);
 822                 pservice->last = now;
 823         }
 824         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 825         if (!ret)
 826                 return;
 827
 828         pr_info("%s: power on\n", dev_name(pservice->dev));
 829
 830 #define BIT_VCODEC_CLK_SEL      (1<<10)
 831         if (cpu_is_rk312x())
 832                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 833                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 834                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 835
 836 #if VCODEC_CLOCK_ENABLE
 837         if (pservice->aclk_vcodec)
 838                 clk_prepare_enable(pservice->aclk_vcodec);
 839         if (pservice->hclk_vcodec)
 840                 clk_prepare_enable(pservice->hclk_vcodec);
 841         if (pservice->clk_core)
 842                 clk_prepare_enable(pservice->clk_core);
 843         if (pservice->clk_cabac)
 844                 clk_prepare_enable(pservice->clk_cabac);
 845         if (pservice->pd_video)
 846                 clk_prepare_enable(pservice->pd_video);
 847 #endif
 848         pm_runtime_get_sync(pservice->dev);
 849
 850         udelay(5);
 851         atomic_add(1, &pservice->power_on_cnt);
 852         wake_lock(&pservice->wake_lock);
 853 }
 854
 855 static inline bool reg_check_interlace(struct vpu_reg *reg)
 856 {
 857         u32 type = (reg->reg[3] & (1 << 23));
 858
 859         return (type > 0);
 860 }
 861
 862 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 863 {
 864         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 865
 866         return type;
 867 }
 868
 869 static inline int reg_probe_width(struct vpu_reg *reg)
 870 {
 871         int width_in_mb = reg->reg[4] >> 23;
 872
 873         return width_in_mb * 16;
 874 }
 875
 876 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 877 {
 878         int y_virstride = reg->reg[8];
 879
 880         return y_virstride;
 881 }
 882
 883 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 884                              struct vpu_reg *reg, int fd)
 885 {
 886         struct vpu_service_info *pservice = data->pservice;
 887         struct ion_handle *hdl;
 888         int ret = 0;
 889         struct vcodec_mem_region *mem_region;
 890
 891         hdl = ion_import_dma_buf(pservice->ion_client, fd);
 892         if (IS_ERR(hdl)) {
 893                 vpu_err("import dma-buf from fd %d failed\n", fd);
 894                 return PTR_ERR(hdl);
 895         }
 896         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 897
 898         if (mem_region == NULL) {
 899                 vpu_err("allocate memory for iommu memory region failed\n");
 900                 ion_free(pservice->ion_client, hdl);
 901                 return -1;
 902         }
 903
 904         mem_region->hdl = hdl;
 905         if (data->mmu_dev)
 906                 ret = ion_map_iommu(data->dev, pservice->ion_client,
 907                                     mem_region->hdl, &mem_region->iova,
 908                                     &mem_region->len);
 909         else
 910                 ret = ion_phys(pservice->ion_client,
 911                                mem_region->hdl,
 912                                (ion_phys_addr_t *)&mem_region->iova,
 913                                (size_t *)&mem_region->len);
 914
 915         if (ret < 0) {
 916                 vpu_err("fd %d ion map iommu failed\n", fd);
 917                 kfree(mem_region);
 918                 ion_free(pservice->ion_client, hdl);
 919                 return ret;
 920         }
 921         INIT_LIST_HEAD(&mem_region->reg_lnk);
 922         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 923         return mem_region->iova;
 924 }
 925
 926 /*
 927  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 928  * it by pps id in video stream data.
 929  *
 930  * So we need to translate the address in iommu case. The address data is also
 931  * 10bit fd + 22bit offset mode.
 932  * Because userspace decoder do not give the pps id in the register file sets
 933  * kernel driver need to translate each scaling list address in pps buffer which
 934  * means 256 pps for H.264, 64 pps for H.265.
 935  *
 936  * In order to optimize the performance kernel driver ask userspace decoder to
 937  * set all scaling list address in pps buffer to the same one which will be used
 938  * on current decoding task. Then kernel driver can only translate the first
 939  * address then copy it all pps buffer.
 940  */
 941 static void fill_scaling_list_addr_in_pps(
 942                 struct vpu_subdev_data *data,
 943                 struct vpu_reg *reg,
 944                 char *pps,
 945                 int pps_info_count,
 946                 int pps_info_size,
 947                 int scaling_list_addr_offset)
 948 {
 949         int base = scaling_list_addr_offset;
 950         int scaling_fd = 0;
 951         u32 scaling_offset;
 952
 953         scaling_offset  = (u32)pps[base + 0];
 954         scaling_offset += (u32)pps[base + 1] << 8;
 955         scaling_offset += (u32)pps[base + 2] << 16;
 956         scaling_offset += (u32)pps[base + 3] << 24;
 957
 958         scaling_fd = scaling_offset & 0x3ff;
 959         scaling_offset = scaling_offset >> 10;
 960
 961         if (scaling_fd > 0) {
 962                 int i = 0;
 963                 u32 tmp = vcodec_fd_to_iova(data, reg, scaling_fd);
 964                 tmp += scaling_offset;
 965
 966                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 967                         pps[base + 0] = (tmp >>  0) & 0xff;
 968                         pps[base + 1] = (tmp >>  8) & 0xff;
 969                         pps[base + 2] = (tmp >> 16) & 0xff;
 970                         pps[base + 3] = (tmp >> 24) & 0xff;
 971                 }
 972         }
 973 }
 974
 975 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data, const u8 *tbl,
 976                                 int size, struct vpu_reg *reg,
 977                                 struct extra_info_for_iommu *ext_inf)
 978 {
 979         struct vpu_service_info *pservice = data->pservice;
 980         struct vpu_task_info *task = reg->task;
 981         enum FORMAT_TYPE type;
 982         struct ion_handle *hdl;
 983         int ret = 0;
 984         struct vcodec_mem_region *mem_region;
 985         int i;
 986         int offset = 0;
 987
 988         if (tbl == NULL || size <= 0) {
 989                 dev_err(pservice->dev, "input arguments invalidate\n");
 990                 return -1;
 991         }
 992
 993         if (task->get_fmt)
 994                 type = task->get_fmt(reg->reg);
 995         else {
 996                 pr_err("invalid task with NULL get_fmt\n");
 997                 return -1;
 998         }
 999
1000         for (i = 0; i < size; i++) {
1001                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
1002
1003                 /* if userspace do not set the fd at this register, skip */
1004                 if (usr_fd == 0)
1005                         continue;
1006
1007                 /*
1008                  * special offset scale case
1009                  *
1010                  * This translation is for fd + offset translation.
1011                  * One register has 32bits. We need to transfer both buffer file
1012                  * handle and the start address offset so we packet file handle
1013                  * and offset together using below format.
1014                  *
1015                  *  0~9  bit for buffer file handle range 0 ~ 1023
1016                  * 10~31 bit for offset range 0 ~ 4M
1017                  *
1018                  * But on 4K case the offset can be larger the 4M
1019                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1020                  * But MPEG4 will use the same register for colmv and it do not
1021                  * need scale.
1022                  *
1023                  * RKVdec do not have this issue.
1024                  */
1025                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1026                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1027                         offset = reg->reg[tbl[i]] >> 10 << 4;
1028                 else
1029                         offset = reg->reg[tbl[i]] >> 10;
1030
1031                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d\n",
1032                           tbl[i], usr_fd, offset);
1033
1034                 hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
1035                 if (IS_ERR(hdl)) {
1036                         dev_err(pservice->dev,
1037                                 "import dma-buf from fd %d failed, reg[%d]\n",
1038                                 usr_fd, tbl[i]);
1039                         return PTR_ERR(hdl);
1040                 }
1041
1042                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1043                         int pps_info_offset;
1044                         int pps_info_count;
1045                         int pps_info_size;
1046                         int scaling_list_addr_offset;
1047
1048                         switch (type) {
1049                         case FMT_H264D: {
1050                                 pps_info_offset = offset;
1051                                 pps_info_count = 256;
1052                                 pps_info_size = 32;
1053                                 scaling_list_addr_offset = 23;
1054                         } break;
1055                         case FMT_H265D: {
1056                                 pps_info_offset = 0;
1057                                 pps_info_count = 64;
1058                                 pps_info_size = 80;
1059                                 scaling_list_addr_offset = 74;
1060                         } break;
1061                         default: {
1062                                 pps_info_offset = 0;
1063                                 pps_info_count = 0;
1064                                 pps_info_size = 0;
1065                                 scaling_list_addr_offset = 0;
1066                         } break;
1067                         }
1068
1069                         vpu_debug(DEBUG_PPS_FILL,
1070                                   "scaling list filling parameter:\n");
1071                         vpu_debug(DEBUG_PPS_FILL,
1072                                   "pps_info_offset %d\n", pps_info_offset);
1073                         vpu_debug(DEBUG_PPS_FILL,
1074                                   "pps_info_count  %d\n", pps_info_count);
1075                         vpu_debug(DEBUG_PPS_FILL,
1076                                   "pps_info_size   %d\n", pps_info_size);
1077                         vpu_debug(DEBUG_PPS_FILL,
1078                                   "scaling_list_addr_offset %d\n",
1079                                   scaling_list_addr_offset);
1080
1081                         if (pps_info_count) {
1082                                 char *pps = (char *)ion_map_kernel(
1083                                                 pservice->ion_client, hdl);
1084                                 vpu_debug(DEBUG_PPS_FILL,
1085                                           "scaling list setting pps %p\n", pps);
1086                                 pps += pps_info_offset;
1087
1088                                 fill_scaling_list_addr_in_pps(
1089                                                 data, reg, pps,
1090                                                 pps_info_count,
1091                                                 pps_info_size,
1092                                                 scaling_list_addr_offset);
1093                         }
1094                 }
1095
1096                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1097
1098                 if (!mem_region) {
1099                         ion_free(pservice->ion_client, hdl);
1100                         return -1;
1101                 }
1102
1103                 mem_region->hdl = hdl;
1104                 mem_region->reg_idx = tbl[i];
1105
1106                 if (data->mmu_dev)
1107                         ret = ion_map_iommu(data->dev,
1108                                             pservice->ion_client,
1109                                             mem_region->hdl,
1110                                             &mem_region->iova,
1111                                             &mem_region->len);
1112                 else
1113                         ret = ion_phys(pservice->ion_client,
1114                                        mem_region->hdl,
1115                                        (ion_phys_addr_t *)&mem_region->iova,
1116                                        (size_t *)&mem_region->len);
1117
1118                 if (ret < 0) {
1119                         dev_err(pservice->dev, "reg %d fd %d ion map iommu failed\n",
1120                                 tbl[i], usr_fd);
1121                         kfree(mem_region);
1122                         ion_free(pservice->ion_client, hdl);
1123                         return ret;
1124                 }
1125
1126                 /*
1127                  * special for vpu dec num 12: record decoded length
1128                  * hacking for decoded length
1129                  * NOTE: not a perfect fix, the fd is not recorded
1130                  */
1131                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1132                         reg->dec_base = mem_region->iova + offset;
1133                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1134                                   reg->dec_base);
1135                 }
1136
1137                 reg->reg[tbl[i]] = mem_region->iova + offset;
1138                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1139                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1140         }
1141
1142         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1143                 for (i = 0; i < ext_inf->cnt; i++) {
1144                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1145                                   ext_inf->elem[i].index,
1146                                   ext_inf->elem[i].offset);
1147                         reg->reg[ext_inf->elem[i].index] +=
1148                                 ext_inf->elem[i].offset;
1149                 }
1150         }
1151
1152         return 0;
1153 }
1154
1155 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1156                                         struct vpu_reg *reg,
1157                                         struct extra_info_for_iommu *ext_inf)
1158 {
1159         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1160
1161         if (type < FMT_TYPE_BUTT) {
1162                 const struct vpu_trans_info *info = &reg->trans[type];
1163                 const u8 *tbl = info->table;
1164                 int size = info->count;
1165
1166                 return vcodec_bufid_to_iova(data, tbl, size, reg, ext_inf);
1167         }
1168         pr_err("found invalid format type!\n");
1169         return -1;
1170 }
1171
1172 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1173 {
1174
1175         if (!soc_is_rk2928g()) {
1176                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1177                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1178                                 if (reg_probe_width(reg) > 3200) {
1179                                         /*raise frequency for 4k avc.*/
1180                                         reg->freq = VPU_FREQ_600M;
1181                                 }
1182                         } else {
1183                                 if (reg_check_interlace(reg))
1184                                         reg->freq = VPU_FREQ_400M;
1185                         }
1186                 }
1187                 if (data->hw_id == HEVC_ID) {
1188                         if (reg_probe_hevc_y_stride(reg) > 60000)
1189                                 reg->freq = VPU_FREQ_400M;
1190                 }
1191                 if (reg->type == VPU_PP)
1192                         reg->freq = VPU_FREQ_400M;
1193         }
1194 }
1195
1196 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1197                                 struct vpu_session *session,
1198                                 void __user *src, u32 size)
1199 {
1200         struct vpu_service_info *pservice = data->pservice;
1201         int extra_size = 0;
1202         struct extra_info_for_iommu extra_info;
1203         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1204                                       GFP_KERNEL);
1205
1206         vpu_debug_enter();
1207
1208         if (NULL == reg) {
1209                 vpu_err("error: kmalloc fail in reg_init\n");
1210                 return NULL;
1211         }
1212
1213         if (size > data->reg_size) {
1214                 pr_err("vpu reg size %u is larger than hw reg size %u\n",
1215                        size, data->reg_size);
1216                 extra_size = size - data->reg_size;
1217                 size = data->reg_size;
1218         }
1219         reg->session = session;
1220         reg->data = data;
1221         reg->type = session->type;
1222         reg->size = size;
1223         reg->freq = VPU_FREQ_DEFAULT;
1224         reg->task = &data->task_info[session->type];
1225         reg->trans = data->trans_info;
1226         reg->reg = (u32 *)&reg[1];
1227         INIT_LIST_HEAD(&reg->session_link);
1228         INIT_LIST_HEAD(&reg->status_link);
1229
1230         INIT_LIST_HEAD(&reg->mem_region_list);
1231
1232         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1233                 vpu_err("error: copy_from_user failed in reg_init\n");
1234                 kfree(reg);
1235                 return NULL;
1236         }
1237
1238         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1239                 vpu_err("error: copy_from_user failed in reg_init\n");
1240                 kfree(reg);
1241                 return NULL;
1242         }
1243
1244         if (0 > vcodec_reg_address_translate(data, reg, &extra_info)) {
1245                 int i = 0;
1246
1247                 vpu_err("error: translate reg address failed, dumping regs\n");
1248                 for (i = 0; i < size >> 2; i++)
1249                         pr_err("reg[%02d]: %08x\n", i, *((u32 *)src + i));
1250
1251                 kfree(reg);
1252                 return NULL;
1253         }
1254
1255         mutex_lock(&pservice->lock);
1256         list_add_tail(&reg->status_link, &pservice->waiting);
1257         list_add_tail(&reg->session_link, &session->waiting);
1258         mutex_unlock(&pservice->lock);
1259
1260         if (pservice->auto_freq)
1261                 get_reg_freq(data, reg);
1262
1263         vpu_debug_leave();
1264         return reg;
1265 }
1266
1267 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1268 {
1269         struct vpu_service_info *pservice = data->pservice;
1270         struct vcodec_mem_region *mem_region = NULL, *n;
1271
1272         list_del_init(&reg->session_link);
1273         list_del_init(&reg->status_link);
1274         if (reg == pservice->reg_codec)
1275                 pservice->reg_codec = NULL;
1276         if (reg == pservice->reg_pproc)
1277                 pservice->reg_pproc = NULL;
1278
1279         /* release memory region attach to this registers table. */
1280         list_for_each_entry_safe(mem_region, n,
1281                         &reg->mem_region_list, reg_lnk) {
1282                 ion_free(pservice->ion_client, mem_region->hdl);
1283                 list_del_init(&mem_region->reg_lnk);
1284                 kfree(mem_region);
1285         }
1286
1287         kfree(reg);
1288 }
1289
1290 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1291                                  struct vpu_reg *reg)
1292 {
1293         vpu_debug_enter();
1294         list_del_init(&reg->status_link);
1295         list_add_tail(&reg->status_link, &pservice->running);
1296
1297         list_del_init(&reg->session_link);
1298         list_add_tail(&reg->session_link, &reg->session->running);
1299         vpu_debug_leave();
1300 }
1301
1302 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1303 {
1304         int i;
1305         u32 *dst = reg->reg;
1306
1307         vpu_debug_enter();
1308         for (i = 0; i < count; i++, src++)
1309                 *dst++ = readl_relaxed(src);
1310
1311         dst = (u32 *)&reg->reg[0];
1312         for (i = 0; i < count; i++)
1313                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1314
1315         vpu_debug_leave();
1316 }
1317
1318 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1319                                  struct vpu_reg *reg)
1320 {
1321         struct vpu_service_info *pservice = data->pservice;
1322         struct vpu_hw_info *hw_info = data->hw_info;
1323         struct vpu_task_info *task = reg->task;
1324
1325         vpu_debug_enter();
1326
1327         list_del_init(&reg->status_link);
1328         list_add_tail(&reg->status_link, &pservice->done);
1329
1330         list_del_init(&reg->session_link);
1331         list_add_tail(&reg->session_link, &reg->session->done);
1332
1333         switch (reg->type) {
1334         case VPU_ENC: {
1335                 pservice->reg_codec = NULL;
1336                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1337                 reg->reg[task->reg_irq] = pservice->irq_status;
1338         } break;
1339         case VPU_DEC: {
1340                 pservice->reg_codec = NULL;
1341                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1342
1343                 /* revert hack for decoded length */
1344                 if (task->reg_len > 0) {
1345                         int reg_len = task->reg_len;
1346                         u32 dec_get = reg->reg[reg_len];
1347                         s32 dec_length = dec_get - reg->dec_base;
1348
1349                         vpu_debug(DEBUG_REGISTER,
1350                                   "dec_get %08x dec_length %d\n",
1351                                   dec_get, dec_length);
1352                         reg->reg[reg_len] = dec_length << 10;
1353                 }
1354
1355                 reg->reg[task->reg_irq] = pservice->irq_status;
1356         } break;
1357         case VPU_PP: {
1358                 pservice->reg_pproc = NULL;
1359                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1360                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1361         } break;
1362         case VPU_DEC_PP: {
1363                 u32 pipe_mode;
1364                 u32 *regs = data->dec_dev.regs;
1365
1366                 pservice->reg_codec = NULL;
1367                 pservice->reg_pproc = NULL;
1368
1369                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1370
1371                 /* NOTE: remove pp pipeline mode flag first */
1372                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1373                 pipe_mode &= ~task->pipe_mask;
1374                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1375
1376                 /* revert hack for decoded length */
1377                 if (task->reg_len > 0) {
1378                         int reg_len = task->reg_len;
1379                         u32 dec_get = reg->reg[reg_len];
1380                         s32 dec_length = dec_get - reg->dec_base;
1381
1382                         vpu_debug(DEBUG_REGISTER,
1383                                   "dec_get %08x dec_length %d\n",
1384                                   dec_get, dec_length);
1385                         reg->reg[reg_len] = dec_length << 10;
1386                 }
1387
1388                 reg->reg[task->reg_irq] = pservice->irq_status;
1389         } break;
1390         default: {
1391                 vpu_err("error: copy reg from hw with unknown type %d\n",
1392                         reg->type);
1393         } break;
1394         }
1395         vcodec_exit_mode(data);
1396
1397         atomic_sub(1, &reg->session->task_running);
1398         atomic_sub(1, &pservice->total_running);
1399         wake_up(&reg->session->wait);
1400
1401         vpu_debug_leave();
1402 }
1403
1404 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1405                                  struct vpu_reg *reg)
1406 {
1407         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1408
1409         if (curr == reg->freq)
1410                 return;
1411
1412         atomic_set(&pservice->freq_status, reg->freq);
1413         switch (reg->freq) {
1414         case VPU_FREQ_200M: {
1415                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1416         } break;
1417         case VPU_FREQ_266M: {
1418                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1419         } break;
1420         case VPU_FREQ_300M: {
1421                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1422         } break;
1423         case VPU_FREQ_400M: {
1424                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1425         } break;
1426         case VPU_FREQ_500M: {
1427                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1428         } break;
1429         case VPU_FREQ_600M: {
1430                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1431         } break;
1432         default: {
1433                 unsigned long rate = 300*MHZ;
1434
1435                 if (soc_is_rk2928g())
1436                         rate = 400*MHZ;
1437
1438                 clk_set_rate(pservice->aclk_vcodec, rate);
1439         } break;
1440         }
1441 }
1442
1443 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1444 {
1445         struct vpu_service_info *pservice = data->pservice;
1446         struct vpu_task_info *task = reg->task;
1447         struct vpu_hw_info *hw_info = data->hw_info;
1448         int i;
1449         u32 *src = (u32 *)&reg->reg[0];
1450         u32 enable_mask = task->enable_mask;
1451         u32 gating_mask = task->gating_mask;
1452         u32 reg_en = task->reg_en;
1453
1454         vpu_debug_enter();
1455
1456         atomic_add(1, &pservice->total_running);
1457         atomic_add(1, &reg->session->task_running);
1458
1459         if (pservice->auto_freq)
1460                 vpu_service_set_freq(pservice, reg);
1461
1462         vcodec_enter_mode(data);
1463
1464         switch (reg->type) {
1465         case VPU_ENC: {
1466                 u32 *dst = data->enc_dev.regs;
1467                 u32 base = 0;
1468                 u32 end  = hw_info->enc_reg_num;
1469                 /* u32 reg_gating = task->reg_gating; */
1470
1471                 pservice->reg_codec = reg;
1472
1473                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1474                           base, end, reg_en, enable_mask, gating_mask);
1475
1476                 VEPU_CLEAN_CACHE(dst);
1477
1478                 if (debug & DEBUG_SET_REG)
1479                         for (i = base; i < end; i++)
1480                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1481                                           i, src[i]);
1482
1483                 /*
1484                  * NOTE: encoder need to setup mode first
1485                  */
1486                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1487
1488                 /* NOTE: encoder gating is not on enable register */
1489                 /* src[reg_gating] |= gating_mask; */
1490
1491                 for (i = base; i < end; i++) {
1492                         if (i != reg_en)
1493                                 writel_relaxed(src[i], dst + i);
1494                 }
1495
1496                 writel(src[reg_en], dst + reg_en);
1497                 dsb(sy);
1498
1499                 time_record(reg->task, 0);
1500         } break;
1501         case VPU_DEC: {
1502                 u32 *dst = data->dec_dev.regs;
1503                 u32 len = hw_info->dec_reg_num;
1504                 u32 base = hw_info->base_dec;
1505                 u32 end  = hw_info->end_dec;
1506
1507                 pservice->reg_codec = reg;
1508
1509                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1510                           base, end, reg_en, enable_mask, gating_mask);
1511
1512                 VDPU_CLEAN_CACHE(dst);
1513
1514                 /* on rkvdec set cache size to 64byte */
1515                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1516                         u32 *cache_base = dst + 0x100;
1517                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1518                         writel_relaxed(val, cache_base + 0x07);
1519                         writel_relaxed(val, cache_base + 0x17);
1520                 }
1521
1522                 if (debug & DEBUG_SET_REG)
1523                         for (i = 0; i < len; i++)
1524                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1525                                           i, src[i]);
1526
1527                 /*
1528                  * NOTE: The end register is invalid. Do NOT write to it
1529                  *       Also the base register must be written
1530                  */
1531                 for (i = base; i < end; i++) {
1532                         if (i != reg_en)
1533                                 writel_relaxed(src[i], dst + i);
1534                 }
1535
1536                 writel(src[reg_en] | gating_mask, dst + reg_en);
1537                 dsb(sy);
1538
1539                 time_record(reg->task, 0);
1540         } break;
1541         case VPU_PP: {
1542                 u32 *dst = data->dec_dev.regs;
1543                 u32 base = hw_info->base_pp;
1544                 u32 end  = hw_info->end_pp;
1545
1546                 pservice->reg_pproc = reg;
1547
1548                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1549                           base, end, reg_en, enable_mask, gating_mask);
1550
1551                 if (debug & DEBUG_SET_REG)
1552                         for (i = base; i < end; i++)
1553                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1554                                           i, src[i]);
1555
1556                 for (i = base; i < end; i++) {
1557                         if (i != reg_en)
1558                                 writel_relaxed(src[i], dst + i);
1559                 }
1560
1561                 writel(src[reg_en] | gating_mask, dst + reg_en);
1562                 dsb(sy);
1563
1564                 time_record(reg->task, 0);
1565         } break;
1566         case VPU_DEC_PP: {
1567                 u32 *dst = data->dec_dev.regs;
1568                 u32 base = hw_info->base_dec_pp;
1569                 u32 end  = hw_info->end_dec_pp;
1570
1571                 pservice->reg_codec = reg;
1572                 pservice->reg_pproc = reg;
1573
1574                 vpu_debug(DEBUG_TASK_INFO, "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1575                           base, end, reg_en, enable_mask, gating_mask);
1576
1577                 /* VDPU_SOFT_RESET(dst); */
1578                 VDPU_CLEAN_CACHE(dst);
1579
1580                 if (debug & DEBUG_SET_REG)
1581                         for (i = base; i < end; i++)
1582                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1583                                           i, src[i]);
1584
1585                 for (i = base; i < end; i++) {
1586                         if (i != reg_en)
1587                                 writel_relaxed(src[i], dst + i);
1588                 }
1589
1590                 /* NOTE: dec output must be disabled */
1591
1592                 writel(src[reg_en] | gating_mask, dst + reg_en);
1593                 dsb(sy);
1594
1595                 time_record(reg->task, 0);
1596         } break;
1597         default: {
1598                 vpu_err("error: unsupport session type %d", reg->type);
1599                 atomic_sub(1, &pservice->total_running);
1600                 atomic_sub(1, &reg->session->task_running);
1601         } break;
1602         }
1603
1604         vpu_debug_leave();
1605 }
1606
1607 static void try_set_reg(struct vpu_subdev_data *data)
1608 {
1609         struct vpu_service_info *pservice = data->pservice;
1610
1611         vpu_debug_enter();
1612
1613         mutex_lock(&pservice->shutdown_lock);
1614         if (atomic_read(&pservice->service_on) == 0) {
1615                 mutex_lock(&pservice->shutdown_lock);
1616                 return;
1617         }
1618         if (!list_empty(&pservice->waiting)) {
1619                 struct vpu_reg *reg_codec = pservice->reg_codec;
1620                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1621                 int can_set = 0;
1622                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1623                 int reset_request = atomic_read(&pservice->reset_request);
1624                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1625                                 struct vpu_reg, status_link);
1626
1627                 vpu_service_power_on(pservice);
1628
1629                 if (change_able || !reset_request) {
1630                         switch (reg->type) {
1631                         case VPU_ENC: {
1632                                 if (change_able)
1633                                         can_set = 1;
1634                         } break;
1635                         case VPU_DEC: {
1636                                 if (reg_codec == NULL)
1637                                         can_set = 1;
1638                                 if (pservice->auto_freq && (reg_pproc != NULL))
1639                                         can_set = 0;
1640                         } break;
1641                         case VPU_PP: {
1642                                 if (reg_codec == NULL) {
1643                                         if (reg_pproc == NULL)
1644                                                 can_set = 1;
1645                                 } else {
1646                                         if ((reg_codec->type == VPU_DEC) &&
1647                                             (reg_pproc == NULL))
1648                                                 can_set = 1;
1649
1650                                         /*
1651                                          * NOTE:
1652                                          * can not charge frequency
1653                                          * when vpu is working
1654                                          */
1655                                         if (pservice->auto_freq)
1656                                                 can_set = 0;
1657                                 }
1658                         } break;
1659                         case VPU_DEC_PP: {
1660                                 if (change_able)
1661                                         can_set = 1;
1662                                 } break;
1663                         default: {
1664                                 pr_err("undefined reg type %d\n", reg->type);
1665                         } break;
1666                         }
1667                 }
1668
1669                 /* then check reset request */
1670                 if (reset_request && !change_able)
1671                         reset_request = 0;
1672
1673                 /* do reset before setting registers */
1674                 if (reset_request)
1675                         vpu_reset(data);
1676
1677                 if (can_set) {
1678                         reg_from_wait_to_run(pservice, reg);
1679                         reg_copy_to_hw(reg->data, reg);
1680                 }
1681         }
1682
1683         mutex_unlock(&pservice->shutdown_lock);
1684         vpu_debug_leave();
1685 }
1686
1687 static int return_reg(struct vpu_subdev_data *data,
1688                       struct vpu_reg *reg, u32 __user *dst)
1689 {
1690         struct vpu_hw_info *hw_info = data->hw_info;
1691         size_t size = reg->size;
1692         u32 base;
1693
1694         vpu_debug_enter();
1695         switch (reg->type) {
1696         case VPU_ENC: {
1697                 base = 0;
1698         } break;
1699         case VPU_DEC: {
1700                 base = hw_info->base_dec_pp;
1701         } break;
1702         case VPU_PP: {
1703                 base = hw_info->base_pp;
1704         } break;
1705         case VPU_DEC_PP: {
1706                 base = hw_info->base_dec_pp;
1707         } break;
1708         default: {
1709                 vpu_err("error: copy reg to user with unknown type %d\n",
1710                         reg->type);
1711                 return -EFAULT;
1712         } break;
1713         }
1714
1715         if (copy_to_user(dst, &reg->reg[base], size)) {
1716                 vpu_err("error: return_reg copy_to_user failed\n");
1717                 return -EFAULT;
1718         }
1719
1720         reg_deinit(data, reg);
1721         vpu_debug_leave();
1722         return 0;
1723 }
1724
1725 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1726                               unsigned long arg)
1727 {
1728         struct vpu_subdev_data *data =
1729                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1730                              struct vpu_subdev_data, cdev);
1731         struct vpu_service_info *pservice = data->pservice;
1732         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1733
1734         vpu_debug_enter();
1735         if (NULL == session)
1736                 return -EINVAL;
1737
1738         switch (cmd) {
1739         case VPU_IOC_SET_CLIENT_TYPE: {
1740                 session->type = (enum VPU_CLIENT_TYPE)arg;
1741                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1742                           session->pid, session->type);
1743         } break;
1744         case VPU_IOC_GET_HW_FUSE_STATUS: {
1745                 struct vpu_request req;
1746
1747                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1748                           session->pid, session->type);
1749                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1750                         vpu_err("error: get hw status copy_from_user failed\n");
1751                         return -EFAULT;
1752                 } else {
1753                         void *config = (session->type != VPU_ENC) ?
1754                                        ((void *)&pservice->dec_config) :
1755                                        ((void *)&pservice->enc_config);
1756                         size_t size = (session->type != VPU_ENC) ?
1757                                       (sizeof(struct vpu_dec_config)) :
1758                                       (sizeof(struct vpu_enc_config));
1759                         if (copy_to_user((void __user *)req.req,
1760                                          config, size)) {
1761                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1762                                         session->type);
1763                                 return -EFAULT;
1764                         }
1765                 }
1766         } break;
1767         case VPU_IOC_SET_REG: {
1768                 struct vpu_request req;
1769                 struct vpu_reg *reg;
1770
1771                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1772                           session->pid, session->type);
1773                 if (copy_from_user(&req, (void __user *)arg,
1774                                    sizeof(struct vpu_request))) {
1775                         vpu_err("error: set reg copy_from_user failed\n");
1776                         return -EFAULT;
1777                 }
1778
1779                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1780                 if (NULL == reg) {
1781                         return -EFAULT;
1782                 } else {
1783                         mutex_lock(&pservice->lock);
1784                         try_set_reg(data);
1785                         mutex_unlock(&pservice->lock);
1786                 }
1787         } break;
1788         case VPU_IOC_GET_REG: {
1789                 struct vpu_request req;
1790                 struct vpu_reg *reg;
1791                 int ret;
1792
1793                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1794                           session->pid, session->type);
1795                 if (copy_from_user(&req, (void __user *)arg,
1796                                    sizeof(struct vpu_request))) {
1797                         vpu_err("error: get reg copy_from_user failed\n");
1798                         return -EFAULT;
1799                 }
1800
1801                 ret = wait_event_timeout(session->wait,
1802                                          !list_empty(&session->done),
1803                                          VPU_TIMEOUT_DELAY);
1804
1805                 if (!list_empty(&session->done)) {
1806                         if (ret < 0)
1807                                 vpu_err("warning: pid %d wait task error ret %d\n",
1808                                         session->pid, ret);
1809                         ret = 0;
1810                 } else {
1811                         if (unlikely(ret < 0)) {
1812                                 vpu_err("error: pid %d wait task ret %d\n",
1813                                         session->pid, ret);
1814                         } else if (ret == 0) {
1815                                 vpu_err("error: pid %d wait %d task done timeout\n",
1816                                         session->pid,
1817                                         atomic_read(&session->task_running));
1818                                 ret = -ETIMEDOUT;
1819                         }
1820                 }
1821
1822                 if (ret < 0) {
1823                         int task_running = atomic_read(&session->task_running);
1824
1825                         mutex_lock(&pservice->lock);
1826                         vpu_service_dump(pservice);
1827                         if (task_running) {
1828                                 atomic_set(&session->task_running, 0);
1829                                 atomic_sub(task_running,
1830                                            &pservice->total_running);
1831                                 pr_err("%d task is running but not return, reset hardware...",
1832                                        task_running);
1833                                 vpu_reset(data);
1834                                 pr_err("done\n");
1835                         }
1836                         vpu_service_session_clear(data, session);
1837                         mutex_unlock(&pservice->lock);
1838                         return ret;
1839                 }
1840
1841                 mutex_lock(&pservice->lock);
1842                 reg = list_entry(session->done.next,
1843                                  struct vpu_reg, session_link);
1844                 return_reg(data, reg, (u32 __user *)req.req);
1845                 mutex_unlock(&pservice->lock);
1846         } break;
1847         case VPU_IOC_PROBE_IOMMU_STATUS: {
1848                 int iommu_enable = 1;
1849
1850                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1851
1852                 if (copy_to_user((void __user *)arg,
1853                                  &iommu_enable, sizeof(int))) {
1854                         vpu_err("error: iommu status copy_to_user failed\n");
1855                         return -EFAULT;
1856                 }
1857         } break;
1858         default: {
1859                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1860         } break;
1861         }
1862         vpu_debug_leave();
1863         return 0;
1864 }
1865
1866 #ifdef CONFIG_COMPAT
1867 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1868                                      unsigned long arg)
1869 {
1870         struct vpu_subdev_data *data =
1871                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1872                              struct vpu_subdev_data, cdev);
1873         struct vpu_service_info *pservice = data->pservice;
1874         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1875
1876         vpu_debug_enter();
1877         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1878                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1879         if (NULL == session)
1880                 return -EINVAL;
1881
1882         switch (cmd) {
1883         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1884                 session->type = (enum VPU_CLIENT_TYPE)arg;
1885                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1886                           session->type);
1887         } break;
1888         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1889                 struct compat_vpu_request req;
1890
1891                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1892                           session->type);
1893                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1894                                    sizeof(struct compat_vpu_request))) {
1895                         vpu_err("error: compat get hw status copy_from_user failed\n");
1896                         return -EFAULT;
1897                 } else {
1898                         void *config = (session->type != VPU_ENC) ?
1899                                        ((void *)&pservice->dec_config) :
1900                                        ((void *)&pservice->enc_config);
1901                         size_t size = (session->type != VPU_ENC) ?
1902                                       (sizeof(struct vpu_dec_config)) :
1903                                       (sizeof(struct vpu_enc_config));
1904
1905                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1906                                          config, size)) {
1907                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1908                                         session->type);
1909                                 return -EFAULT;
1910                         }
1911                 }
1912         } break;
1913         case COMPAT_VPU_IOC_SET_REG: {
1914                 struct compat_vpu_request req;
1915                 struct vpu_reg *reg;
1916
1917                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1918                           session->type);
1919                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1920                                    sizeof(struct compat_vpu_request))) {
1921                         vpu_err("compat set_reg copy_from_user failed\n");
1922                         return -EFAULT;
1923                 }
1924                 reg = reg_init(data, session,
1925                                compat_ptr((compat_uptr_t)req.req), req.size);
1926                 if (NULL == reg) {
1927                         return -EFAULT;
1928                 } else {
1929                         mutex_lock(&pservice->lock);
1930                         try_set_reg(data);
1931                         mutex_unlock(&pservice->lock);
1932                 }
1933         } break;
1934         case COMPAT_VPU_IOC_GET_REG: {
1935                 struct compat_vpu_request req;
1936                 struct vpu_reg *reg;
1937                 int ret;
1938
1939                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1940                           session->type);
1941                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1942                                    sizeof(struct compat_vpu_request))) {
1943                         vpu_err("compat get reg copy_from_user failed\n");
1944                         return -EFAULT;
1945                 }
1946
1947                 ret = wait_event_timeout(session->wait,
1948                                          !list_empty(&session->done),
1949                                          VPU_TIMEOUT_DELAY);
1950
1951                 if (!list_empty(&session->done)) {
1952                         if (ret < 0)
1953                                 vpu_err("warning: pid %d wait task error ret %d\n",
1954                                         session->pid, ret);
1955                         ret = 0;
1956                 } else {
1957                         if (unlikely(ret < 0)) {
1958                                 vpu_err("error: pid %d wait task ret %d\n",
1959                                         session->pid, ret);
1960                         } else if (ret == 0) {
1961                                 vpu_err("error: pid %d wait %d task done timeout\n",
1962                                         session->pid,
1963                                         atomic_read(&session->task_running));
1964                                 ret = -ETIMEDOUT;
1965                         }
1966                 }
1967
1968                 if (ret < 0) {
1969                         int task_running = atomic_read(&session->task_running);
1970
1971                         mutex_lock(&pservice->lock);
1972                         vpu_service_dump(pservice);
1973                         if (task_running) {
1974                                 atomic_set(&session->task_running, 0);
1975                                 atomic_sub(task_running,
1976                                            &pservice->total_running);
1977                                 pr_err("%d task is running but not return, reset hardware...",
1978                                        task_running);
1979                                 vpu_reset(data);
1980                                 pr_err("done\n");
1981                         }
1982                         vpu_service_session_clear(data, session);
1983                         mutex_unlock(&pservice->lock);
1984                         return ret;
1985                 }
1986
1987                 mutex_lock(&pservice->lock);
1988                 reg = list_entry(session->done.next,
1989                                  struct vpu_reg, session_link);
1990                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
1991                 mutex_unlock(&pservice->lock);
1992         } break;
1993         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
1994                 int iommu_enable = 1;
1995
1996                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
1997
1998                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
1999                                  &iommu_enable, sizeof(int))) {
2000                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2001                         return -EFAULT;
2002                 }
2003         } break;
2004         default: {
2005                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2006         } break;
2007         }
2008         vpu_debug_leave();
2009         return 0;
2010 }
2011 #endif
2012
2013 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2014 {
2015         int ret = -EINVAL, i = 0;
2016         u32 hw_id = readl_relaxed(data->regs);
2017
2018         hw_id = (hw_id >> 16) & 0xFFFF;
2019         pr_info("checking hw id %x\n", hw_id);
2020         data->hw_info = NULL;
2021         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2022                 struct vcodec_info *info = &vcodec_info_set[i];
2023
2024                 if (hw_id == info->hw_id) {
2025                         data->hw_id = info->hw_id;
2026                         data->hw_info = info->hw_info;
2027                         data->task_info = info->task_info;
2028                         data->trans_info = info->trans_info;
2029                         ret = 0;
2030                         break;
2031                 }
2032         }
2033         return ret;
2034 }
2035
2036 static int vpu_service_open(struct inode *inode, struct file *filp)
2037 {
2038         struct vpu_subdev_data *data = container_of(
2039                         inode->i_cdev, struct vpu_subdev_data, cdev);
2040         struct vpu_service_info *pservice = data->pservice;
2041         struct vpu_session *session = kmalloc(sizeof(*session), GFP_KERNEL);
2042
2043         vpu_debug_enter();
2044
2045         if (NULL == session) {
2046                 vpu_err("error: unable to allocate memory for vpu_session.");
2047                 return -ENOMEM;
2048         }
2049
2050         session->type   = VPU_TYPE_BUTT;
2051         session->pid    = current->pid;
2052         INIT_LIST_HEAD(&session->waiting);
2053         INIT_LIST_HEAD(&session->running);
2054         INIT_LIST_HEAD(&session->done);
2055         INIT_LIST_HEAD(&session->list_session);
2056         init_waitqueue_head(&session->wait);
2057         atomic_set(&session->task_running, 0);
2058         mutex_lock(&pservice->lock);
2059         list_add_tail(&session->list_session, &pservice->session);
2060         filp->private_data = (void *)session;
2061         mutex_unlock(&pservice->lock);
2062
2063         pr_debug("dev opened\n");
2064         vpu_debug_leave();
2065         return nonseekable_open(inode, filp);
2066 }
2067
2068 static int vpu_service_release(struct inode *inode, struct file *filp)
2069 {
2070         struct vpu_subdev_data *data = container_of(
2071                         inode->i_cdev, struct vpu_subdev_data, cdev);
2072         struct vpu_service_info *pservice = data->pservice;
2073         int task_running;
2074         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2075
2076         vpu_debug_enter();
2077         if (NULL == session)
2078                 return -EINVAL;
2079
2080         task_running = atomic_read(&session->task_running);
2081         if (task_running) {
2082                 pr_err("error: session %d still has %d task running when closing\n",
2083                        session->pid, task_running);
2084                 msleep(50);
2085         }
2086         wake_up(&session->wait);
2087
2088         mutex_lock(&pservice->lock);
2089         /* remove this filp from the asynchronusly notified filp's */
2090         list_del_init(&session->list_session);
2091         vpu_service_session_clear(data, session);
2092         kfree(session);
2093         filp->private_data = NULL;
2094         mutex_unlock(&pservice->lock);
2095
2096         pr_debug("dev closed\n");
2097         vpu_debug_leave();
2098         return 0;
2099 }
2100
2101 static const struct file_operations vpu_service_fops = {
2102         .unlocked_ioctl = vpu_service_ioctl,
2103         .open           = vpu_service_open,
2104         .release        = vpu_service_release,
2105 #ifdef CONFIG_COMPAT
2106         .compat_ioctl   = compat_vpu_service_ioctl,
2107 #endif
2108 };
2109
2110 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2111 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2112 static irqreturn_t vepu_irq(int irq, void *dev_id);
2113 static irqreturn_t vepu_isr(int irq, void *dev_id);
2114 static void get_hw_info(struct vpu_subdev_data *data);
2115
2116 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2117 {
2118         struct device_node *dn = NULL;
2119         struct platform_device *pd = NULL;
2120         struct device *ret = NULL;
2121
2122         dn = of_find_compatible_node(NULL, NULL, compt);
2123         if (!dn) {
2124                 pr_err("can't find device node %s \r\n", compt);
2125                 return NULL;
2126         }
2127
2128         pd = of_find_device_by_node(dn);
2129         if (!pd) {
2130                 pr_err("can't find platform device in device node %s\n", compt);
2131                 return  NULL;
2132         }
2133         ret = &pd->dev;
2134
2135         return ret;
2136 }
2137
2138 #ifdef CONFIG_IOMMU_API
2139 static inline void platform_set_sysmmu(struct device *iommu,
2140                                        struct device *dev)
2141 {
2142         dev->archdata.iommu = iommu;
2143 }
2144 #else
2145 static inline void platform_set_sysmmu(struct device *iommu,
2146                                        struct device *dev)
2147 {
2148 }
2149 #endif
2150
2151 int vcodec_sysmmu_fault_hdl(struct device *dev,
2152                             enum rk_iommu_inttype itype,
2153                             unsigned long pgtable_base,
2154                             unsigned long fault_addr, unsigned int status)
2155 {
2156         struct platform_device *pdev;
2157         struct vpu_service_info *pservice;
2158         struct vpu_subdev_data *data;
2159
2160         vpu_debug_enter();
2161
2162         if (dev == NULL) {
2163                 pr_err("invalid NULL dev\n");
2164                 return 0;
2165         }
2166
2167         pdev = container_of(dev, struct platform_device, dev);
2168         if (pdev == NULL) {
2169                 pr_err("invalid NULL platform_device\n");
2170                 return 0;
2171         }
2172
2173         data = platform_get_drvdata(pdev);
2174         if (data == NULL) {
2175                 pr_err("invalid NULL vpu_subdev_data\n");
2176                 return 0;
2177         }
2178
2179         pservice = data->pservice;
2180         if (pservice == NULL) {
2181                 pr_err("invalid NULL vpu_service_info\n");
2182                 return 0;
2183         }
2184
2185         if (pservice->reg_codec) {
2186                 struct vpu_reg *reg = pservice->reg_codec;
2187                 struct vcodec_mem_region *mem, *n;
2188                 int i = 0;
2189
2190                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2191                 if (!list_empty(&reg->mem_region_list)) {
2192                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2193                                                  reg_lnk) {
2194                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2195                                        mem->reg_idx, i, mem->iova, mem->len);
2196                                 i++;
2197                         }
2198                 } else {
2199                         pr_err("no memory region mapped\n");
2200                 }
2201
2202                 if (reg->data) {
2203                         struct vpu_subdev_data *data = reg->data;
2204                         u32 *base = (u32 *)data->dec_dev.regs;
2205                         u32 len = data->hw_info->dec_reg_num;
2206
2207                         pr_err("current errror register set:\n");
2208
2209                         for (i = 0; i < len; i++)
2210                                 pr_err("reg[%02d] %08x\n",
2211                                        i, readl_relaxed(base + i));
2212                 }
2213
2214                 pr_alert("vcodec, page fault occur, reset hw\n");
2215
2216                 /* reg->reg[101] = 1; */
2217                 vpu_reset(data);
2218         }
2219
2220         return 0;
2221 }
2222
2223 static int vcodec_subdev_probe(struct platform_device *pdev,
2224                                struct vpu_service_info *pservice)
2225 {
2226         int ret = 0;
2227         struct resource *res = NULL;
2228         u32 ioaddr = 0;
2229         u8 *regs = NULL;
2230         struct vpu_hw_info *hw_info = NULL;
2231         struct device *dev = &pdev->dev;
2232         char *name = (char *)dev_name(dev);
2233         struct device_node *np = pdev->dev.of_node;
2234         struct vpu_subdev_data *data =
2235                 devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2236         u32 iommu_en = 0;
2237         char mmu_dev_dts_name[40];
2238
2239         of_property_read_u32(np, "iommu_enabled", &iommu_en);
2240
2241         pr_info("probe device %s\n", dev_name(dev));
2242
2243         data->pservice = pservice;
2244         data->dev = dev;
2245
2246         of_property_read_string(np, "name", (const char **)&name);
2247         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2248
2249         if (pservice->reg_base == 0) {
2250                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2251                 data->regs = devm_ioremap_resource(dev, res);
2252                 if (IS_ERR(data->regs)) {
2253                         ret = PTR_ERR(data->regs);
2254                         goto err;
2255                 }
2256                 ioaddr = res->start;
2257         } else {
2258                 data->regs = pservice->reg_base;
2259                 ioaddr = pservice->ioaddr;
2260         }
2261
2262         clear_bit(MMU_ACTIVATED, &data->state);
2263         vcodec_enter_mode(data);
2264
2265         vpu_service_power_on(pservice);
2266         ret = vpu_service_check_hw(data);
2267         if (ret < 0) {
2268                 vpu_err("error: hw info check faild\n");
2269                 goto err;
2270         }
2271
2272         hw_info = data->hw_info;
2273         regs = (u8 *)data->regs;
2274
2275         if (hw_info->dec_reg_num) {
2276                 data->dec_dev.iosize = hw_info->dec_io_size;
2277                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2278         }
2279
2280         if (hw_info->enc_reg_num) {
2281                 data->enc_dev.iosize = hw_info->enc_io_size;
2282                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2283         }
2284
2285         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2286
2287         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2288         if (data->irq_enc > 0) {
2289                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2290                                                 vepu_irq, vepu_isr,
2291                                                 IRQF_SHARED, dev_name(dev),
2292                                                 (void *)data);
2293                 if (ret) {
2294                         dev_err(dev, "error: can't request vepu irq %d\n",
2295                                 data->irq_enc);
2296                         goto err;
2297                 }
2298         }
2299         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2300         if (data->irq_dec > 0) {
2301                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2302                                                 vdpu_irq, vdpu_isr,
2303                                                 IRQF_SHARED, dev_name(dev),
2304                                                 (void *)data);
2305                 if (ret) {
2306                         dev_err(dev, "error: can't request vdpu irq %d\n",
2307                                 data->irq_dec);
2308                         goto err;
2309                 }
2310         }
2311         atomic_set(&data->dec_dev.irq_count_codec, 0);
2312         atomic_set(&data->dec_dev.irq_count_pp, 0);
2313         atomic_set(&data->enc_dev.irq_count_codec, 0);
2314         atomic_set(&data->enc_dev.irq_count_pp, 0);
2315
2316         if (iommu_en) {
2317                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
2318                         sprintf(mmu_dev_dts_name,
2319                                 HEVC_IOMMU_COMPATIBLE_NAME);
2320                 else if (data->mode == VCODEC_RUNNING_MODE_VPU)
2321                         sprintf(mmu_dev_dts_name,
2322                                 VPU_IOMMU_COMPATIBLE_NAME);
2323                 else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2324                         sprintf(mmu_dev_dts_name, VDEC_IOMMU_COMPATIBLE_NAME);
2325                 else
2326                         sprintf(mmu_dev_dts_name,
2327                                 HEVC_IOMMU_COMPATIBLE_NAME);
2328
2329                 data->mmu_dev =
2330                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2331
2332                 if (data->mmu_dev)
2333                         platform_set_sysmmu(data->mmu_dev, dev);
2334
2335                 rockchip_iovmm_set_fault_handler(dev, vcodec_sysmmu_fault_hdl);
2336         }
2337
2338         get_hw_info(data);
2339         pservice->auto_freq = true;
2340
2341         vcodec_exit_mode(data);
2342         /* create device node */
2343         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2344         if (ret) {
2345                 dev_err(dev, "alloc dev_t failed\n");
2346                 goto err;
2347         }
2348
2349         cdev_init(&data->cdev, &vpu_service_fops);
2350
2351         data->cdev.owner = THIS_MODULE;
2352         data->cdev.ops = &vpu_service_fops;
2353
2354         ret = cdev_add(&data->cdev, data->dev_t, 1);
2355
2356         if (ret) {
2357                 dev_err(dev, "add dev_t failed\n");
2358                 goto err;
2359         }
2360
2361         data->cls = class_create(THIS_MODULE, name);
2362
2363         if (IS_ERR(data->cls)) {
2364                 ret = PTR_ERR(data->cls);
2365                 dev_err(dev, "class_create err:%d\n", ret);
2366                 goto err;
2367         }
2368
2369         data->child_dev = device_create(data->cls, dev,
2370                 data->dev_t, NULL, name);
2371
2372         platform_set_drvdata(pdev, data);
2373
2374         INIT_LIST_HEAD(&data->lnk_service);
2375         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2376
2377 #ifdef CONFIG_DEBUG_FS
2378         data->debugfs_dir = vcodec_debugfs_create_device_dir(name, parent);
2379         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2380                 data->debugfs_file_regs =
2381                         debugfs_create_file("regs", 0664, data->debugfs_dir,
2382                                         data, &debug_vcodec_fops);
2383         else
2384                 vpu_err("create debugfs dir %s failed\n", name);
2385 #endif
2386         return 0;
2387 err:
2388         if (data->child_dev) {
2389                 device_destroy(data->cls, data->dev_t);
2390                 cdev_del(&data->cdev);
2391                 unregister_chrdev_region(data->dev_t, 1);
2392         }
2393
2394         if (data->cls)
2395                 class_destroy(data->cls);
2396         return -1;
2397 }
2398
2399 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2400 {
2401         struct vpu_service_info *pservice = data->pservice;
2402
2403         mutex_lock(&pservice->lock);
2404         cancel_delayed_work_sync(&pservice->power_off_work);
2405         vpu_service_power_off(pservice);
2406         mutex_unlock(&pservice->lock);
2407
2408         device_destroy(data->cls, data->dev_t);
2409         class_destroy(data->cls);
2410         cdev_del(&data->cdev);
2411         unregister_chrdev_region(data->dev_t, 1);
2412
2413 #ifdef CONFIG_DEBUG_FS
2414         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2415                 debugfs_remove_recursive(data->debugfs_dir);
2416 #endif
2417 }
2418
2419 static void vcodec_read_property(struct device_node *np,
2420                                  struct vpu_service_info *pservice)
2421 {
2422         pservice->mode_bit = 0;
2423         pservice->mode_ctrl = 0;
2424         pservice->subcnt = 0;
2425         pservice->grf_base = NULL;
2426
2427         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2428
2429         if (pservice->subcnt > 1) {
2430                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2431                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2432         }
2433 #ifdef CONFIG_MFD_SYSCON
2434         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2435         if (IS_ERR_OR_NULL(pservice->grf)) {
2436                 pservice->grf = NULL;
2437 #ifdef CONFIG_ARM
2438                 pservice->grf_base = RK_GRF_VIRT;
2439 #else
2440                 vpu_err("can't find vpu grf property\n");
2441                 return;
2442 #endif
2443         }
2444 #else
2445 #ifdef CONFIG_ARM
2446         pservice->grf_base = RK_GRF_VIRT;
2447 #else
2448         vpu_err("can't find vpu grf property\n");
2449         return;
2450 #endif
2451 #endif
2452
2453 #ifdef CONFIG_RESET_CONTROLLER
2454         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2455         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2456         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2457
2458         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2459                 pr_warn("No aclk reset resource define\n");
2460                 pservice->rst_a = NULL;
2461         }
2462
2463         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2464                 pr_warn("No hclk reset resource define\n");
2465                 pservice->rst_h = NULL;
2466         }
2467
2468         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2469                 pr_warn("No core reset resource define\n");
2470                 pservice->rst_v = NULL;
2471         }
2472 #endif
2473
2474         of_property_read_string(np, "name", (const char **)&pservice->name);
2475 }
2476
2477 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2478 {
2479         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2480         pservice->curr_mode = -1;
2481
2482         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2483         INIT_LIST_HEAD(&pservice->waiting);
2484         INIT_LIST_HEAD(&pservice->running);
2485         mutex_init(&pservice->lock);
2486         mutex_init(&pservice->shutdown_lock);
2487         atomic_set(&pservice->service_on, 1);
2488
2489         INIT_LIST_HEAD(&pservice->done);
2490         INIT_LIST_HEAD(&pservice->session);
2491         INIT_LIST_HEAD(&pservice->subdev_list);
2492
2493         pservice->reg_pproc     = NULL;
2494         atomic_set(&pservice->total_running, 0);
2495         atomic_set(&pservice->enabled,       0);
2496         atomic_set(&pservice->power_on_cnt,  0);
2497         atomic_set(&pservice->power_off_cnt, 0);
2498         atomic_set(&pservice->reset_request, 0);
2499
2500         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2501         pservice->last.tv64 = 0;
2502
2503         pservice->ion_client = rockchip_ion_client_create("vpu");
2504         if (IS_ERR(pservice->ion_client)) {
2505                 vpu_err("failed to create ion client for vcodec ret %ld\n",
2506                         PTR_ERR(pservice->ion_client));
2507         } else {
2508                 vpu_debug(DEBUG_IOMMU, "vcodec ion client create success!\n");
2509         }
2510 }
2511
2512 static int vcodec_probe(struct platform_device *pdev)
2513 {
2514         int i;
2515         int ret = 0;
2516         struct resource *res = NULL;
2517         struct device *dev = &pdev->dev;
2518         struct device_node *np = pdev->dev.of_node;
2519         struct vpu_service_info *pservice =
2520                 devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
2521
2522         pservice->dev = dev;
2523
2524         vcodec_read_property(np, pservice);
2525         vcodec_init_drvdata(pservice);
2526
2527         if (strncmp(pservice->name, "hevc_service", 12) == 0)
2528                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2529         else if (strncmp(pservice->name, "vpu_service", 11) == 0)
2530                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2531         else if (strncmp(pservice->name, "rkvdec", 6) == 0)
2532                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2533         else
2534                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2535
2536         if (0 > vpu_get_clk(pservice))
2537                 goto err;
2538
2539         if (of_property_read_bool(np, "reg")) {
2540                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2541
2542                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2543                 if (IS_ERR(pservice->reg_base)) {
2544                         vpu_err("ioremap registers base failed\n");
2545                         ret = PTR_ERR(pservice->reg_base);
2546                         goto err;
2547                 }
2548                 pservice->ioaddr = res->start;
2549         } else {
2550                 pservice->reg_base = 0;
2551         }
2552
2553         pm_runtime_enable(dev);
2554
2555         if (of_property_read_bool(np, "subcnt")) {
2556                 for (i = 0; i < pservice->subcnt; i++) {
2557                         struct device_node *sub_np;
2558                         struct platform_device *sub_pdev;
2559
2560                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2561                         sub_pdev = of_find_device_by_node(sub_np);
2562
2563                         vcodec_subdev_probe(sub_pdev, pservice);
2564                 }
2565         } else {
2566                 vcodec_subdev_probe(pdev, pservice);
2567         }
2568
2569         vpu_service_power_off(pservice);
2570
2571         pr_info("init success\n");
2572
2573         return 0;
2574
2575 err:
2576         pr_info("init failed\n");
2577         vpu_service_power_off(pservice);
2578         vpu_put_clk(pservice);
2579         wake_lock_destroy(&pservice->wake_lock);
2580
2581         return ret;
2582 }
2583
2584 static int vcodec_remove(struct platform_device *pdev)
2585 {
2586         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2587
2588         vcodec_subdev_remove(data);
2589
2590         pm_runtime_disable(data->pservice->dev);
2591
2592         return 0;
2593 }
2594
2595 static void vcodec_shutdown(struct platform_device *pdev)
2596 {
2597         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2598         struct vpu_service_info *pservice = data->pservice;
2599
2600         dev_info(&pdev->dev, "%s IN\n", __func__);
2601
2602         mutex_lock(&pservice->shutdown_lock);
2603         atomic_set(&pservice->service_on, 0);
2604         mutex_unlock(&pservice->shutdown_lock);
2605
2606         vcodec_exit_mode(data);
2607
2608         vpu_service_clear(data);
2609         vcodec_subdev_remove(data);
2610
2611         pm_runtime_disable(&pdev->dev);
2612 }
2613
2614 #if defined(CONFIG_OF)
2615 static const struct of_device_id vcodec_service_dt_ids[] = {
2616         {.compatible = "rockchip,vpu_service",},
2617         {.compatible = "rockchip,hevc_service",},
2618         {.compatible = "rockchip,vpu_combo",},
2619         {.compatible = "rockchip,rkvdec",},
2620         {},
2621 };
2622 #endif
2623
2624 static struct platform_driver vcodec_driver = {
2625         .probe = vcodec_probe,
2626         .remove = vcodec_remove,
2627         .shutdown = vcodec_shutdown,
2628         .driver = {
2629                 .name = "vcodec",
2630                 .owner = THIS_MODULE,
2631 #if defined(CONFIG_OF)
2632                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2633 #endif
2634         },
2635 };
2636
2637 static void get_hw_info(struct vpu_subdev_data *data)
2638 {
2639         struct vpu_service_info *pservice = data->pservice;
2640         struct vpu_dec_config *dec = &pservice->dec_config;
2641         struct vpu_enc_config *enc = &pservice->enc_config;
2642
2643         if (cpu_is_rk2928() || cpu_is_rk3036() ||
2644             cpu_is_rk30xx() || cpu_is_rk312x() ||
2645             cpu_is_rk3188())
2646                 dec->max_dec_pic_width = 1920;
2647         else
2648                 dec->max_dec_pic_width = 4096;
2649
2650         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2651                 dec->h264_support = 3;
2652                 dec->jpeg_support = 1;
2653                 dec->mpeg4_support = 2;
2654                 dec->vc1_support = 3;
2655                 dec->mpeg2_support = 1;
2656                 dec->pp_support = 1;
2657                 dec->sorenson_support = 1;
2658                 dec->ref_buf_support = 3;
2659                 dec->vp6_support = 1;
2660                 dec->vp7_support = 1;
2661                 dec->vp8_support = 1;
2662                 dec->avs_support = 1;
2663                 dec->jpeg_ext_support = 0;
2664                 dec->custom_mpeg4_support = 1;
2665                 dec->reserve = 0;
2666                 dec->mvc_support = 1;
2667
2668                 if (!cpu_is_rk3036()) {
2669                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2670
2671                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2672                         enc->h264_enabled = 1;
2673                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2674                         enc->jpeg_enabled = 1;
2675                         enc->vs_enabled = (config_reg >> 24) & 1;
2676                         enc->rgb_enabled = (config_reg >> 28) & 1;
2677                         enc->reg_size = data->reg_size;
2678                         enc->reserv[0] = 0;
2679                         enc->reserv[1] = 0;
2680                 }
2681
2682                 pservice->auto_freq = true;
2683                 vpu_debug(DEBUG_EXTRA_INFO, "vpu_service set to auto frequency mode\n");
2684                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2685
2686                 pservice->bug_dec_addr = cpu_is_rk30xx();
2687         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2688                 pservice->auto_freq = true;
2689                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2690         } else {
2691                 /* disable frequency switch in hevc.*/
2692                 pservice->auto_freq = false;
2693         }
2694 }
2695
2696 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2697 {
2698         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2699                   task->name, irq_status, task->error_mask);
2700
2701         return (task->error_mask & irq_status) ? true : false;
2702 }
2703
2704 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2705 {
2706         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2707         struct vpu_service_info *pservice = data->pservice;
2708         struct vpu_task_info *task = NULL;
2709         struct vpu_device *dev = &data->dec_dev;
2710         u32 hw_id = data->hw_info->hw_id;
2711         u32 raw_status;
2712         u32 dec_status;
2713
2714         task = &data->task_info[TASK_DEC];
2715
2716         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2717         dec_status = raw_status;
2718
2719         vpu_debug(DEBUG_TASK_INFO, "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2720                   task->reg_irq, dec_status,
2721                   task->irq_mask, task->ready_mask, task->error_mask);
2722
2723         if (dec_status & task->irq_mask) {
2724                 time_record(task, 1);
2725                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2726                           dec_status);
2727                 if ((dec_status & 0x40001) == 0x40001) {
2728                         do {
2729                                 dec_status =
2730                                         readl_relaxed(dev->regs +
2731                                                 task->reg_irq);
2732                         } while ((dec_status & 0x40001) == 0x40001);
2733                 }
2734
2735                 if (check_irq_err(task, dec_status))
2736                         atomic_add(1, &pservice->reset_request);
2737
2738                 writel_relaxed(0, dev->regs + task->reg_irq);
2739
2740                 /*
2741                  * NOTE: rkvdec need to reset after each task to avoid timeout
2742                  *       error on H.264 switch to H.265
2743                  */
2744                 if (data->mode == VCODEC_RUNNING_MODE_RKVDEC)
2745                         writel(0x100000, dev->regs + task->reg_irq);
2746
2747                 /* set clock gating to save power */
2748                 writel(task->gating_mask, dev->regs + task->reg_irq);
2749
2750                 atomic_add(1, &dev->irq_count_codec);
2751                 time_diff(task);
2752         }
2753
2754         task = &data->task_info[TASK_PP];
2755         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2756                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2757
2758                 if (pp_status & task->irq_mask) {
2759                         time_record(task, 1);
2760                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2761                                   pp_status);
2762
2763                         if (check_irq_err(task, dec_status))
2764                                 atomic_add(1, &pservice->reset_request);
2765
2766                         /* clear pp IRQ */
2767                         writel_relaxed(pp_status & (~task->reg_irq),
2768                                        dev->regs + task->irq_mask);
2769                         atomic_add(1, &dev->irq_count_pp);
2770                         time_diff(task);
2771                 }
2772         }
2773
2774         pservice->irq_status = raw_status;
2775
2776         if (atomic_read(&dev->irq_count_pp) ||
2777             atomic_read(&dev->irq_count_codec))
2778                 return IRQ_WAKE_THREAD;
2779         else
2780                 return IRQ_NONE;
2781 }
2782
2783 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2784 {
2785         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2786         struct vpu_service_info *pservice = data->pservice;
2787         struct vpu_device *dev = &data->dec_dev;
2788
2789         mutex_lock(&pservice->lock);
2790         if (atomic_read(&dev->irq_count_codec)) {
2791                 atomic_sub(1, &dev->irq_count_codec);
2792                 if (pservice->reg_codec == NULL) {
2793                         vpu_err("error: dec isr with no task waiting\n");
2794                 } else {
2795                         reg_from_run_to_done(data, pservice->reg_codec);
2796                         /* avoid vpu timeout and can't recover problem */
2797                         VDPU_SOFT_RESET(data->regs);
2798                 }
2799         }
2800
2801         if (atomic_read(&dev->irq_count_pp)) {
2802                 atomic_sub(1, &dev->irq_count_pp);
2803                 if (pservice->reg_pproc == NULL)
2804                         vpu_err("error: pp isr with no task waiting\n");
2805                 else
2806                         reg_from_run_to_done(data, pservice->reg_pproc);
2807         }
2808         try_set_reg(data);
2809         mutex_unlock(&pservice->lock);
2810         return IRQ_HANDLED;
2811 }
2812
2813 static irqreturn_t vepu_irq(int irq, void *dev_id)
2814 {
2815         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2816         struct vpu_service_info *pservice = data->pservice;
2817         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2818         struct vpu_device *dev = &data->enc_dev;
2819         u32 irq_status;
2820
2821         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2822
2823         vpu_debug(DEBUG_TASK_INFO, "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2824                   task->reg_irq, irq_status,
2825                   task->irq_mask, task->ready_mask, task->error_mask);
2826
2827         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2828
2829         if (likely(irq_status & task->irq_mask)) {
2830                 time_record(task, 1);
2831
2832                 if (check_irq_err(task, irq_status))
2833                         atomic_add(1, &pservice->reset_request);
2834
2835                 /* clear enc IRQ */
2836                 writel_relaxed(irq_status & (~task->irq_mask),
2837                                dev->regs + task->reg_irq);
2838
2839                 atomic_add(1, &dev->irq_count_codec);
2840                 time_diff(task);
2841         }
2842
2843         pservice->irq_status = irq_status;
2844
2845         if (atomic_read(&dev->irq_count_codec))
2846                 return IRQ_WAKE_THREAD;
2847         else
2848                 return IRQ_NONE;
2849 }
2850
2851 static irqreturn_t vepu_isr(int irq, void *dev_id)
2852 {
2853         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2854         struct vpu_service_info *pservice = data->pservice;
2855         struct vpu_device *dev = &data->enc_dev;
2856
2857         mutex_lock(&pservice->lock);
2858         if (atomic_read(&dev->irq_count_codec)) {
2859                 atomic_sub(1, &dev->irq_count_codec);
2860                 if (NULL == pservice->reg_codec)
2861                         vpu_err("error: enc isr with no task waiting\n");
2862                 else
2863                         reg_from_run_to_done(data, pservice->reg_codec);
2864         }
2865         try_set_reg(data);
2866         mutex_unlock(&pservice->lock);
2867         return IRQ_HANDLED;
2868 }
2869
2870 static int __init vcodec_service_init(void)
2871 {
2872         int ret = platform_driver_register(&vcodec_driver);
2873
2874         if (ret) {
2875                 vpu_err("Platform device register failed (%d).\n", ret);
2876                 return ret;
2877         }
2878
2879 #ifdef CONFIG_DEBUG_FS
2880         vcodec_debugfs_init();
2881 #endif
2882
2883         return ret;
2884 }
2885
2886 static void __exit vcodec_service_exit(void)
2887 {
2888 #ifdef CONFIG_DEBUG_FS
2889         vcodec_debugfs_exit();
2890 #endif
2891
2892         platform_driver_unregister(&vcodec_driver);
2893 }
2894
2895 module_init(vcodec_service_init);
2896 module_exit(vcodec_service_exit);
2897 MODULE_LICENSE("Proprietary");
2898
2899 #ifdef CONFIG_DEBUG_FS
2900 #include <linux/seq_file.h>
2901
2902 static int vcodec_debugfs_init(void)
2903 {
2904         parent = debugfs_create_dir("vcodec", NULL);
2905         if (!parent)
2906                 return -1;
2907
2908         return 0;
2909 }
2910
2911 static void vcodec_debugfs_exit(void)
2912 {
2913         debugfs_remove(parent);
2914 }
2915
2916 static struct dentry *vcodec_debugfs_create_device_dir(
2917                 char *dirname, struct dentry *parent)
2918 {
2919         return debugfs_create_dir(dirname, parent);
2920 }
2921
2922 static int debug_vcodec_show(struct seq_file *s, void *unused)
2923 {
2924         struct vpu_subdev_data *data = s->private;
2925         struct vpu_service_info *pservice = data->pservice;
2926         unsigned int i, n;
2927         struct vpu_reg *reg, *reg_tmp;
2928         struct vpu_session *session, *session_tmp;
2929
2930         mutex_lock(&pservice->lock);
2931         vpu_service_power_on(pservice);
2932         if (data->hw_info->hw_id != HEVC_ID) {
2933                 seq_puts(s, "\nENC Registers:\n");
2934                 n = data->enc_dev.iosize >> 2;
2935
2936                 for (i = 0; i < n; i++)
2937                         seq_printf(s, "\tswreg%d = %08X\n", i,
2938                                    readl_relaxed(data->enc_dev.regs + i));
2939         }
2940
2941         seq_puts(s, "\nDEC Registers:\n");
2942
2943         n = data->dec_dev.iosize >> 2;
2944         for (i = 0; i < n; i++)
2945                 seq_printf(s, "\tswreg%d = %08X\n", i,
2946                            readl_relaxed(data->dec_dev.regs + i));
2947
2948         seq_puts(s, "\nvpu service status:\n");
2949
2950         list_for_each_entry_safe(session, session_tmp,
2951                                  &pservice->session, list_session) {
2952                 seq_printf(s, "session pid %d type %d:\n",
2953                            session->pid, session->type);
2954
2955                 list_for_each_entry_safe(reg, reg_tmp,
2956                                          &session->waiting, session_link) {
2957                         seq_printf(s, "waiting register set %p\n", reg);
2958                 }
2959                 list_for_each_entry_safe(reg, reg_tmp,
2960                                          &session->running, session_link) {
2961                         seq_printf(s, "running register set %p\n", reg);
2962                 }
2963                 list_for_each_entry_safe(reg, reg_tmp,
2964                                          &session->done, session_link) {
2965                         seq_printf(s, "done    register set %p\n", reg);
2966                 }
2967         }
2968
2969         seq_printf(s, "\npower counter: on %d off %d\n",
2970                    atomic_read(&pservice->power_on_cnt),
2971                    atomic_read(&pservice->power_off_cnt));
2972
2973         mutex_unlock(&pservice->lock);
2974         vpu_service_power_off(pservice);
2975
2976         return 0;
2977 }
2978
2979 static int debug_vcodec_open(struct inode *inode, struct file *file)
2980 {
2981         return single_open(file, debug_vcodec_show, inode->i_private);
2982 }
2983
2984 #endif
2985