drivers/media/platform/rockchip-vpu/rockchip_vpu_enc.c

   1 /*
   2  * Rockchip VPU codec driver
   3  *
   4  * Copyright (C) 2014 Rockchip Electronics Co., Ltd.
   5  *      Alpha Lin <Alpha.Lin@rock-chips.com>
   6  *      Jeffy Chen <jeffy.chen@rock-chips.com>
   7  *
   8  * Copyright (C) 2014 Google, Inc.
   9  *      Tomasz Figa <tfiga@chromium.org>
  10  *
  11  * Based on s5p-mfc driver by Samsung Electronics Co., Ltd.
  12  *
  13  * Copyright (C) 2010-2011 Samsung Electronics Co., Ltd.
  14  *
  15  * This software is licensed under the terms of the GNU General Public
  16  * License version 2, as published by the Free Software Foundation, and
  17  * may be copied, distributed, and modified under those terms.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include "rockchip_vpu_common.h"
  26
  27 #include <linux/clk.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/io.h>
  30 #include <linux/module.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/sched.h>
  33 #include <linux/version.h>
  34 #include <linux/videodev2.h>
  35 #include <media/v4l2-event.h>
  36 #include <linux/workqueue.h>
  37 #include <media/v4l2-ctrls.h>
  38 #include <media/videobuf2-core.h>
  39 #include <media/videobuf2-dma-sg.h>
  40
  41 #include "rockchip_vpu_enc.h"
  42 #include "rockchip_vpu_hw.h"
  43
  44 #define DEF_SRC_FMT_ENC                         V4L2_PIX_FMT_NV12
  45 #define DEF_DST_FMT_ENC                         V4L2_PIX_FMT_VP8
  46
  47 #define ROCKCHIP_ENC_MIN_WIDTH                  96U
  48 #define ROCKCHIP_ENC_MAX_WIDTH                  1920U
  49 #define ROCKCHIP_ENC_MIN_HEIGHT                 96U
  50 #define ROCKCHIP_ENC_MAX_HEIGHT                 1088U
  51
  52 #define V4L2_CID_PRIVATE_ROCKCHIP_HEADER        (V4L2_CID_CUSTOM_BASE + 0)
  53 #define V4L2_CID_PRIVATE_ROCKCHIP_REG_PARAMS    (V4L2_CID_CUSTOM_BASE + 1)
  54 #define V4L2_CID_PRIVATE_ROCKCHIP_HW_PARAMS     (V4L2_CID_CUSTOM_BASE + 2)
  55 #define V4L2_CID_PRIVATE_ROCKCHIP_RET_PARAMS    (V4L2_CID_CUSTOM_BASE + 3)
  56
  57 static struct rockchip_vpu_fmt formats[] = {
  58         /* Source formats. */
  59         {
  60                 .name = "4:2:0 3 planes Y/Cb/Cr",
  61                 .fourcc = V4L2_PIX_FMT_YUV420M,
  62                 .codec_mode = RK_VPU_CODEC_NONE,
  63                 .num_planes = 3,
  64                 .depth = { 8, 4, 4 },
  65                 .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420P,
  66         },
  67         {
  68                 .name = "4:2:0 2 plane Y/CbCr",
  69                 .fourcc = V4L2_PIX_FMT_NV12M,
  70                 .codec_mode = RK_VPU_CODEC_NONE,
  71                 .num_planes = 2,
  72                 .depth = { 8, 8 },
  73                 .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420SP,
  74         },
  75         {
  76                 .name = "4:2:2 1 plane YUYV",
  77                 .fourcc = V4L2_PIX_FMT_YUYV,
  78                 .codec_mode = RK_VPU_CODEC_NONE,
  79                 .num_planes = 1,
  80                 .depth = { 16 },
  81                 .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUYV422,
  82         },
  83         {
  84                 .name = "4:2:2 1 plane UYVY",
  85                 .fourcc = V4L2_PIX_FMT_UYVY,
  86                 .codec_mode = RK_VPU_CODEC_NONE,
  87                 .num_planes = 1,
  88                 .depth = { 16 },
  89                 .enc_fmt = ROCKCHIP_VPU_ENC_FMT_UYVY422,
  90         },
  91         /* Destination formats. */
  92         {
  93                 .name = "VP8 Encoded Stream",
  94                 .fourcc = V4L2_PIX_FMT_VP8,
  95                 .codec_mode = RK3288_VPU_CODEC_VP8E,
  96                 .num_planes = 1,
  97         },
  98 };
  99
 100 static struct rockchip_vpu_fmt *find_format(u32 fourcc, bool bitstream)
 101 {
 102         unsigned int i;
 103
 104         vpu_debug_enter();
 105
 106         for (i = 0; i < ARRAY_SIZE(formats); i++) {
 107                 if (formats[i].fourcc != fourcc)
 108                         continue;
 109                 if (bitstream && formats[i].codec_mode != RK_VPU_CODEC_NONE)
 110                         return &formats[i];
 111                 if (!bitstream && formats[i].codec_mode == RK_VPU_CODEC_NONE)
 112                         return &formats[i];
 113         }
 114
 115         return NULL;
 116 }
 117
 118 /*
 119  * Indices of controls that need to be accessed directly, i.e. through
 120  * p_cur.p pointer of their v4l2_ctrl structs.
 121  */
 122 enum {
 123         ROCKCHIP_VPU_ENC_CTRL_HEADER,
 124         ROCKCHIP_VPU_ENC_CTRL_REG_PARAMS,
 125         ROCKCHIP_VPU_ENC_CTRL_HW_PARAMS,
 126         ROCKCHIP_VPU_ENC_CTRL_RET_PARAMS,
 127 };
 128
 129 static struct rockchip_vpu_control controls[] = {
 130         /* Private, per-frame controls. */
 131         [ROCKCHIP_VPU_ENC_CTRL_HEADER] = {
 132                 .id = V4L2_CID_PRIVATE_ROCKCHIP_HEADER,
 133                 .type = V4L2_CTRL_TYPE_PRIVATE,
 134                 .name = "Rockchip Private Header",
 135                 .elem_size = ROCKCHIP_HEADER_SIZE,
 136                 .max_stores = VIDEO_MAX_FRAME,
 137                 .can_store = true,
 138         },
 139         [ROCKCHIP_VPU_ENC_CTRL_REG_PARAMS] = {
 140                 .id = V4L2_CID_PRIVATE_ROCKCHIP_REG_PARAMS,
 141                 .type = V4L2_CTRL_TYPE_PRIVATE,
 142                 .name = "Rockchip Private Reg Params",
 143                 .elem_size = sizeof(struct rockchip_reg_params),
 144                 .max_stores = VIDEO_MAX_FRAME,
 145                 .can_store = true,
 146         },
 147         [ROCKCHIP_VPU_ENC_CTRL_HW_PARAMS] = {
 148                 .id = V4L2_CID_PRIVATE_ROCKCHIP_HW_PARAMS,
 149                 .type = V4L2_CTRL_TYPE_PRIVATE,
 150                 .name = "Rockchip Private Hw Params",
 151                 .elem_size = ROCKCHIP_HW_PARAMS_SIZE,
 152                 .max_stores = VIDEO_MAX_FRAME,
 153                 .can_store = true,
 154         },
 155         [ROCKCHIP_VPU_ENC_CTRL_RET_PARAMS] = {
 156                 .id = V4L2_CID_PRIVATE_ROCKCHIP_RET_PARAMS,
 157                 .type = V4L2_CTRL_TYPE_PRIVATE,
 158                 .name = "Rockchip Private Ret Params",
 159                 .is_volatile = true,
 160                 .is_read_only = true,
 161                 .max_stores = VIDEO_MAX_FRAME,
 162                 .elem_size = ROCKCHIP_RET_PARAMS_SIZE,
 163         },
 164         /* Generic controls. (currently ignored) */
 165         {
 166                 .id = V4L2_CID_MPEG_VIDEO_GOP_SIZE,
 167                 .type = V4L2_CTRL_TYPE_INTEGER,
 168                 .minimum = 1,
 169                 .maximum = 150,
 170                 .step = 1,
 171                 .default_value = 30,
 172         },
 173         {
 174                 .id = V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE,
 175                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 176                 .minimum = 0,
 177                 .maximum = 1,
 178                 .step = 1,
 179                 .default_value = 0,
 180         },
 181         {
 182                 .id = V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE,
 183                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 184                 .minimum = 0,
 185                 .maximum = 1,
 186                 .step = 1,
 187                 .default_value = 0,
 188         },
 189         {
 190                 .id = V4L2_CID_MPEG_VIDEO_BITRATE,
 191                 .type = V4L2_CTRL_TYPE_INTEGER,
 192                 .minimum = 10000,
 193                 .maximum = 288000000,
 194                 .step = 1,
 195                 .default_value = 2097152,
 196         },
 197         {
 198                 .id = V4L2_CID_MPEG_VIDEO_H264_PROFILE,
 199                 .type = V4L2_CTRL_TYPE_MENU,
 200                 .minimum = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
 201                 .maximum = V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH,
 202                 .default_value = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
 203                 .menu_skip_mask = ~(
 204                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
 205                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
 206                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)
 207                         ),
 208         },
 209         {
 210                 .id = V4L2_CID_MPEG_VIDEO_H264_LEVEL,
 211                 .type = V4L2_CTRL_TYPE_MENU,
 212                 .minimum = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
 213                 .maximum = V4L2_MPEG_VIDEO_H264_LEVEL_4_1,
 214                 .default_value = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
 215         },
 216         {
 217                 .id = V4L2_CID_MPEG_VIDEO_H264_MAX_QP,
 218                 .type = V4L2_CTRL_TYPE_INTEGER,
 219                 .minimum = 0,
 220                 .maximum = 51,
 221                 .step = 1,
 222                 .default_value = 30,
 223         },
 224         {
 225                 .id = V4L2_CID_MPEG_VIDEO_H264_MIN_QP,
 226                 .type = V4L2_CTRL_TYPE_INTEGER,
 227                 .minimum = 0,
 228                 .maximum = 51,
 229                 .step = 1,
 230                 .default_value = 18,
 231         },
 232         {
 233                 .id = V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM,
 234                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 235                 .minimum = 0,
 236                 .maximum = 1,
 237                 .step = 1,
 238                 .default_value = 0,
 239         },
 240         {
 241                 .id = V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE,
 242                 .type = V4L2_CTRL_TYPE_INTEGER,
 243                 .minimum = 0,
 244                 .maximum = 288000,
 245                 .step = 1,
 246                 .default_value = 30000,
 247         },
 248         {
 249                 .id = V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING,
 250                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 251                 .minimum = 0,
 252                 .maximum = 1,
 253                 .step = 1,
 254                 .default_value = 0,
 255         },
 256         {
 257                 .id = V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME,
 258                 .type = V4L2_CTRL_TYPE_BUTTON,
 259         },
 260         /*
 261          * This hardware does not support features provided by controls
 262          * below, but they are required for compatibility with certain
 263          * userspace software.
 264          */
 265         {
 266                 .id = V4L2_CID_MPEG_MFC51_VIDEO_RC_REACTION_COEFF,
 267                 .type = V4L2_CTRL_TYPE_INTEGER,
 268                 .name = "Rate Control Reaction Coeff.",
 269                 .minimum = 1,
 270                 .maximum = (1 << 16) - 1,
 271                 .step = 1,
 272                 .default_value = 1,
 273         },
 274         {
 275                 .id = V4L2_CID_MPEG_VIDEO_HEADER_MODE,
 276                 .type = V4L2_CTRL_TYPE_MENU,
 277                 .minimum = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
 278                 .maximum = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
 279                 .default_value = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
 280                 .menu_skip_mask = 0,
 281         },
 282         {
 283                 .id = V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT,
 284                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 285                 .name = "Fixed Target Bit Enable",
 286                 .minimum = 0,
 287                 .maximum = 1,
 288                 .default_value = 0,
 289                 .step = 1,
 290                 .menu_skip_mask = 0,
 291         },
 292         {
 293                 .id = V4L2_CID_MPEG_VIDEO_B_FRAMES,
 294                 .type = V4L2_CTRL_TYPE_INTEGER,
 295                 .minimum = 0,
 296                 .maximum = 2,
 297                 .step = 1,
 298                 .default_value = 0,
 299         },
 300         {
 301                 .id = V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP,
 302                 .type = V4L2_CTRL_TYPE_INTEGER,
 303                 .minimum = 0,
 304                 .maximum = 51,
 305                 .step = 1,
 306                 .default_value = 1,
 307         },
 308 };
 309
 310 static inline const void *get_ctrl_ptr(struct rockchip_vpu_ctx *ctx, unsigned id)
 311 {
 312         struct v4l2_ctrl *ctrl = ctx->ctrls[id];
 313
 314         return ctrl->p_cur.p;
 315 }
 316
 317 static const char *const *rockchip_vpu_enc_get_menu(u32 id)
 318 {
 319         static const char *const vpu_video_frame_skip[] = {
 320                 "Disabled",
 321                 "Level Limit",
 322                 "VBV/CPB Limit",
 323                 NULL,
 324         };
 325
 326         switch (id) {
 327         case V4L2_CID_MPEG_MFC51_VIDEO_FRAME_SKIP_MODE:
 328                 return vpu_video_frame_skip;
 329         }
 330
 331         return NULL;
 332 }
 333
 334 static int vidioc_querycap(struct file *file, void *priv,
 335                            struct v4l2_capability *cap)
 336 {
 337         struct rockchip_vpu_dev *dev = video_drvdata(file);
 338
 339         vpu_debug_enter();
 340
 341         strlcpy(cap->driver, ROCKCHIP_VPU_ENC_NAME, sizeof(cap->driver));
 342         strlcpy(cap->card, dev->pdev->name, sizeof(cap->card));
 343         strlcpy(cap->bus_info, "platform:" ROCKCHIP_VPU_NAME,
 344                 sizeof(cap->bus_info));
 345
 346         /*
 347          * This is only a mem-to-mem video device. The capture and output
 348          * device capability flags are left only for backward compatibility
 349          * and are scheduled for removal.
 350          */
 351         cap->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING |
 352             V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_OUTPUT_MPLANE;
 353         cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
 354
 355         vpu_debug_leave();
 356
 357         return 0;
 358 }
 359
 360 static int vidioc_enum_framesizes(struct file *file, void *prov,
 361                                   struct v4l2_frmsizeenum *fsize)
 362 {
 363         struct v4l2_frmsize_stepwise *s = &fsize->stepwise;
 364         struct rockchip_vpu_fmt *fmt;
 365
 366         if (fsize->index != 0) {
 367                 vpu_debug(0, "invalid frame size index (expected 0, got %d)\n",
 368                                 fsize->index);
 369                 return -EINVAL;
 370         }
 371
 372         fmt = find_format(fsize->pixel_format, true);
 373         if (!fmt) {
 374                 vpu_debug(0, "unsupported bitstream format (%08x)\n",
 375                                 fsize->pixel_format);
 376                 return -EINVAL;
 377         }
 378
 379         fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
 380
 381         s->min_width = ROCKCHIP_ENC_MIN_WIDTH;
 382         s->max_width = ROCKCHIP_ENC_MAX_WIDTH;
 383         s->step_width = MB_DIM;
 384         s->min_height = ROCKCHIP_ENC_MIN_HEIGHT;
 385         s->max_height = ROCKCHIP_ENC_MAX_HEIGHT;
 386         s->step_height = MB_DIM;
 387
 388         return 0;
 389 }
 390
 391 static int vidioc_enum_fmt(struct v4l2_fmtdesc *f, bool out)
 392 {
 393         struct rockchip_vpu_fmt *fmt;
 394         int i, j = 0;
 395
 396         vpu_debug_enter();
 397
 398         for (i = 0; i < ARRAY_SIZE(formats); ++i) {
 399                 if (out && formats[i].codec_mode != RK_VPU_CODEC_NONE)
 400                         continue;
 401                 else if (!out && formats[i].codec_mode == RK_VPU_CODEC_NONE)
 402                         continue;
 403
 404                 if (j == f->index) {
 405                         fmt = &formats[i];
 406                         strlcpy(f->description, fmt->name,
 407                                 sizeof(f->description));
 408                         f->pixelformat = fmt->fourcc;
 409
 410                         f->flags = 0;
 411                         if (formats[i].codec_mode != RK_VPU_CODEC_NONE)
 412                                 f->flags |= V4L2_FMT_FLAG_COMPRESSED;
 413
 414                         vpu_debug_leave();
 415
 416                         return 0;
 417                 }
 418
 419                 ++j;
 420         }
 421
 422         vpu_debug_leave();
 423
 424         return -EINVAL;
 425 }
 426
 427 static int vidioc_enum_fmt_vid_cap_mplane(struct file *file, void *priv,
 428                                           struct v4l2_fmtdesc *f)
 429 {
 430         return vidioc_enum_fmt(f, false);
 431 }
 432
 433 static int vidioc_enum_fmt_vid_out_mplane(struct file *file, void *priv,
 434                                           struct v4l2_fmtdesc *f)
 435 {
 436         return vidioc_enum_fmt(f, true);
 437 }
 438
 439 static int vidioc_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
 440 {
 441         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 442
 443         vpu_debug_enter();
 444
 445         vpu_debug(4, "f->type = %d\n", f->type);
 446
 447         switch (f->type) {
 448         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 449                 f->fmt.pix_mp = ctx->dst_fmt;
 450                 break;
 451
 452         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 453                 f->fmt.pix_mp = ctx->src_fmt;
 454                 break;
 455
 456         default:
 457                 vpu_err("invalid buf type\n");
 458                 return -EINVAL;
 459         }
 460
 461         vpu_debug_leave();
 462
 463         return 0;
 464 }
 465
 466 static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f)
 467 {
 468         struct rockchip_vpu_fmt *fmt;
 469         struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
 470         char str[5];
 471
 472         vpu_debug_enter();
 473
 474         switch (f->type) {
 475         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 476                 vpu_debug(4, "%s\n", fmt2str(f->fmt.pix_mp.pixelformat, str));
 477
 478                 fmt = find_format(pix_fmt_mp->pixelformat, true);
 479                 if (!fmt) {
 480                         vpu_err("failed to try capture format\n");
 481                         return -EINVAL;
 482                 }
 483
 484                 if (pix_fmt_mp->plane_fmt[0].sizeimage == 0) {
 485                         vpu_err("must be set encoding output size\n");
 486                         return -EINVAL;
 487                 }
 488
 489                 pix_fmt_mp->plane_fmt[0].bytesperline = 0;
 490                 break;
 491
 492         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 493                 vpu_debug(4, "%s\n", fmt2str(f->fmt.pix_mp.pixelformat, str));
 494
 495                 fmt = find_format(pix_fmt_mp->pixelformat, false);
 496                 if (!fmt) {
 497                         vpu_err("failed to try output format\n");
 498                         return -EINVAL;
 499                 }
 500
 501                 if (fmt->num_planes != pix_fmt_mp->num_planes) {
 502                         vpu_err("plane number mismatches on output format\n");
 503                         return -EINVAL;
 504                 }
 505
 506                 /* Limit to hardware min/max. */
 507                 pix_fmt_mp->width = clamp(pix_fmt_mp->width,
 508                                 ROCKCHIP_ENC_MIN_WIDTH, ROCKCHIP_ENC_MAX_WIDTH);
 509                 pix_fmt_mp->height = clamp(pix_fmt_mp->height,
 510                                 ROCKCHIP_ENC_MIN_HEIGHT, ROCKCHIP_ENC_MAX_HEIGHT);
 511                 /* Round up to macroblocks. */
 512                 pix_fmt_mp->width = round_up(pix_fmt_mp->width, MB_DIM);
 513                 pix_fmt_mp->height = round_up(pix_fmt_mp->height, MB_DIM);
 514                 break;
 515
 516         default:
 517                 vpu_err("invalid buf type\n");
 518                 return -EINVAL;
 519         }
 520
 521         vpu_debug_leave();
 522
 523         return 0;
 524 }
 525
 526 static void calculate_plane_sizes(struct rockchip_vpu_fmt *fmt,
 527                                   unsigned int w, unsigned int h,
 528                                   struct v4l2_pix_format_mplane *pix_fmt_mp)
 529 {
 530         int i;
 531
 532         for (i = 0; i < fmt->num_planes; ++i) {
 533                 pix_fmt_mp->plane_fmt[i].bytesperline = w * fmt->depth[i] / 8;
 534                 pix_fmt_mp->plane_fmt[i].sizeimage = h *
 535                                         pix_fmt_mp->plane_fmt[i].bytesperline;
 536                 /*
 537                  * All of multiplanar formats we support have chroma
 538                  * planes subsampled by 2 vertically.
 539                  */
 540                 if (i != 0)
 541                         pix_fmt_mp->plane_fmt[i].sizeimage /= 2;
 542         }
 543 }
 544
 545 static int vidioc_s_fmt(struct file *file, void *priv, struct v4l2_format *f)
 546 {
 547         struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
 548         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 549         unsigned int mb_width, mb_height;
 550         struct rockchip_vpu_fmt *fmt;
 551         int ret = 0;
 552
 553         vpu_debug_enter();
 554
 555         /* Change not allowed if any queue is streaming. */
 556         if (vb2_is_streaming(&ctx->vq_src) || vb2_is_streaming(&ctx->vq_dst)) {
 557                 ret = -EBUSY;
 558                 goto out;
 559         }
 560
 561         switch (f->type) {
 562         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 563                 /*
 564                  * Pixel format change is not allowed when the other queue has
 565                  * buffers allocated.
 566                  */
 567                 if (vb2_is_busy(&ctx->vq_src)
 568                     && pix_fmt_mp->pixelformat != ctx->dst_fmt.pixelformat) {
 569                         ret = -EBUSY;
 570                         goto out;
 571                 }
 572
 573                 ret = vidioc_try_fmt(file, priv, f);
 574                 if (ret)
 575                         goto out;
 576
 577                 ctx->vpu_dst_fmt = find_format(pix_fmt_mp->pixelformat, true);
 578                 ctx->dst_fmt = *pix_fmt_mp;
 579                 break;
 580
 581         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 582                 /*
 583                  * Pixel format change is not allowed when the other queue has
 584                  * buffers allocated.
 585                  */
 586                 if (vb2_is_busy(&ctx->vq_dst)
 587                     && pix_fmt_mp->pixelformat != ctx->src_fmt.pixelformat) {
 588                         ret = -EBUSY;
 589                         goto out;
 590                 }
 591
 592                 ret = vidioc_try_fmt(file, priv, f);
 593                 if (ret)
 594                         goto out;
 595
 596                 fmt = find_format(pix_fmt_mp->pixelformat, false);
 597                 ctx->vpu_src_fmt = fmt;
 598
 599                 mb_width = MB_WIDTH(pix_fmt_mp->width);
 600                 mb_height = MB_HEIGHT(pix_fmt_mp->height);
 601
 602                 vpu_debug(0, "OUTPUT codec mode: %d\n", fmt->codec_mode);
 603                 vpu_debug(0, "fmt - w: %d, h: %d, mb - w: %d, h: %d\n",
 604                           pix_fmt_mp->width, pix_fmt_mp->height,
 605                           mb_width, mb_height);
 606
 607                 calculate_plane_sizes(fmt, mb_width * MB_DIM,
 608                                         mb_height * MB_DIM, pix_fmt_mp);
 609
 610                 /* Reset crop rectangle. */
 611                 ctx->src_crop.width = pix_fmt_mp->width;
 612                 ctx->src_crop.height = pix_fmt_mp->height;
 613
 614                 ctx->src_fmt = *pix_fmt_mp;
 615                 break;
 616
 617         default:
 618                 vpu_err("invalid buf type\n");
 619                 return -EINVAL;
 620         }
 621
 622 out:
 623         vpu_debug_leave();
 624
 625         return ret;
 626 }
 627
 628 static int vidioc_reqbufs(struct file *file, void *priv,
 629                           struct v4l2_requestbuffers *reqbufs)
 630 {
 631         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 632         int ret;
 633
 634         vpu_debug_enter();
 635
 636         switch (reqbufs->type) {
 637         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 638                 vpu_debug(4, "\n");
 639
 640                 ret = vb2_reqbufs(&ctx->vq_dst, reqbufs);
 641                 if (ret != 0) {
 642                         vpu_err("error in vb2_reqbufs() for E(D)\n");
 643                         goto out;
 644                 }
 645                 break;
 646
 647         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 648                 vpu_debug(4, "memory type %d\n", reqbufs->memory);
 649
 650                 ret = vb2_reqbufs(&ctx->vq_src, reqbufs);
 651                 if (ret != 0) {
 652                         vpu_err("error in vb2_reqbufs() for E(S)\n");
 653                         goto out;
 654                 }
 655                 break;
 656
 657         default:
 658                 vpu_err("invalid buf type\n");
 659                 ret = -EINVAL;
 660                 goto out;
 661         }
 662
 663 out:
 664         vpu_debug_leave();
 665
 666         return ret;
 667 }
 668
 669 static int vidioc_querybuf(struct file *file, void *priv,
 670                            struct v4l2_buffer *buf)
 671 {
 672         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 673         int ret;
 674
 675         vpu_debug_enter();
 676
 677         switch (buf->type) {
 678         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 679                 ret = vb2_querybuf(&ctx->vq_dst, buf);
 680                 if (ret != 0) {
 681                         vpu_err("error in vb2_querybuf() for E(D)\n");
 682                         goto out;
 683                 }
 684
 685                 buf->m.planes[0].m.mem_offset += DST_QUEUE_OFF_BASE;
 686                 break;
 687
 688         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 689                 ret = vb2_querybuf(&ctx->vq_src, buf);
 690                 if (ret != 0) {
 691                         vpu_err("error in vb2_querybuf() for E(S)\n");
 692                         goto out;
 693                 }
 694                 break;
 695
 696         default:
 697                 vpu_err("invalid buf type\n");
 698                 ret = -EINVAL;
 699                 goto out;
 700         }
 701
 702 out:
 703         vpu_debug_leave();
 704
 705         return ret;
 706 }
 707
 708 static int vidioc_qbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
 709 {
 710         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 711         int ret;
 712         int i;
 713
 714         vpu_debug_enter();
 715
 716         for (i = 0; i < buf->length; i++) {
 717                 vpu_debug(4, "plane[%d]->length %d bytesused %d\n",
 718                           i, buf->m.planes[i].length,
 719                           buf->m.planes[i].bytesused);
 720         }
 721
 722         switch (buf->type) {
 723         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 724                 ret = vb2_qbuf(&ctx->vq_src, buf);
 725                 vpu_debug(4, "vb2_qbuf return %d\n", ret);
 726                 break;
 727
 728         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 729                 ret = vb2_qbuf(&ctx->vq_dst, buf);
 730                 vpu_debug(4, "vb2_qbuf return %d\n", ret);
 731                 break;
 732
 733         default:
 734                 ret = -EINVAL;
 735         }
 736
 737         vpu_debug_leave();
 738
 739         return ret;
 740 }
 741
 742 static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
 743 {
 744         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 745         int ret;
 746
 747         vpu_debug_enter();
 748
 749         switch (buf->type) {
 750         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 751                 ret = vb2_dqbuf(&ctx->vq_src, buf, file->f_flags & O_NONBLOCK);
 752                 break;
 753
 754         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 755                 ret = vb2_dqbuf(&ctx->vq_dst, buf, file->f_flags & O_NONBLOCK);
 756                 break;
 757
 758         default:
 759                 ret = -EINVAL;
 760         }
 761
 762         vpu_debug_leave();
 763
 764         return ret;
 765 }
 766
 767 static int vidioc_expbuf(struct file *file, void *priv,
 768                          struct v4l2_exportbuffer *eb)
 769 {
 770         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 771         int ret;
 772
 773         vpu_debug_enter();
 774
 775         switch (eb->type) {
 776         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 777                 ret = vb2_expbuf(&ctx->vq_src, eb);
 778                 break;
 779
 780         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 781                 ret = vb2_expbuf(&ctx->vq_dst, eb);
 782                 break;
 783
 784         default:
 785                 ret = -EINVAL;
 786         }
 787
 788         vpu_debug_leave();
 789
 790         return ret;
 791 }
 792
 793 static int vidioc_streamon(struct file *file, void *priv,
 794                            enum v4l2_buf_type type)
 795 {
 796         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 797         int ret;
 798
 799         vpu_debug_enter();
 800
 801         switch (type) {
 802         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 803                 ret = vb2_streamon(&ctx->vq_src, type);
 804                 break;
 805
 806         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 807                 ret = vb2_streamon(&ctx->vq_dst, type);
 808                 break;
 809
 810         default:
 811                 ret = -EINVAL;
 812         }
 813
 814         vpu_debug_leave();
 815
 816         return ret;
 817 }
 818
 819 static int vidioc_streamoff(struct file *file, void *priv,
 820                             enum v4l2_buf_type type)
 821 {
 822         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 823         int ret;
 824
 825         vpu_debug_enter();
 826
 827         switch (type) {
 828         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 829                 ret = vb2_streamoff(&ctx->vq_src, type);
 830                 break;
 831
 832         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 833                 ret = vb2_streamoff(&ctx->vq_dst, type);
 834                 break;
 835
 836         default:
 837                 ret = -EINVAL;
 838         }
 839
 840         vpu_debug_leave();
 841
 842         return ret;
 843 }
 844
 845 static int rockchip_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
 846 {
 847         struct rockchip_vpu_ctx *ctx = ctrl_to_ctx(ctrl);
 848         struct rockchip_vpu_dev *dev = ctx->dev;
 849         int ret = 0;
 850
 851         vpu_debug_enter();
 852
 853         vpu_debug(4, "ctrl id %d\n", ctrl->id);
 854
 855         switch (ctrl->id) {
 856         case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
 857         case V4L2_CID_MPEG_VIDEO_BITRATE:
 858         case V4L2_CID_MPEG_VIDEO_H264_MAX_QP:
 859         case V4L2_CID_MPEG_VIDEO_H264_MIN_QP:
 860         case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE:
 861         case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE:
 862         case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
 863         case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
 864         case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM:
 865         case V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE:
 866         case V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING:
 867         case V4L2_CID_MPEG_MFC51_VIDEO_RC_REACTION_COEFF:
 868         case V4L2_CID_MPEG_VIDEO_HEADER_MODE:
 869         case V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT:
 870         case V4L2_CID_MPEG_VIDEO_B_FRAMES:
 871         case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP:
 872         case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME:
 873                 /* Ignore these controls for now. (FIXME?) */
 874                 break;
 875
 876         case V4L2_CID_PRIVATE_ROCKCHIP_HEADER:
 877         case V4L2_CID_PRIVATE_ROCKCHIP_REG_PARAMS:
 878         case V4L2_CID_PRIVATE_ROCKCHIP_HW_PARAMS:
 879                 /* Nothing to do here. The control is used directly. */
 880                 break;
 881
 882         default:
 883                 v4l2_err(&dev->v4l2_dev, "Invalid control, id=%d, val=%d\n",
 884                          ctrl->id, ctrl->val);
 885                 ret = -EINVAL;
 886         }
 887
 888         vpu_debug_leave();
 889
 890         return ret;
 891 }
 892
 893 static int rockchip_vpu_enc_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
 894 {
 895         struct rockchip_vpu_ctx *ctx = ctrl_to_ctx(ctrl);
 896         struct rockchip_vpu_dev *dev = ctx->dev;
 897         int ret = 0;
 898
 899         vpu_debug_enter();
 900
 901         vpu_debug(4, "ctrl id %d\n", ctrl->id);
 902
 903         switch (ctrl->id) {
 904         case V4L2_CID_PRIVATE_ROCKCHIP_RET_PARAMS:
 905                 memcpy(ctrl->p_new.p, ctx->run.priv_dst.cpu,
 906                         ROCKCHIP_RET_PARAMS_SIZE);
 907                 break;
 908
 909         default:
 910                 v4l2_err(&dev->v4l2_dev, "Invalid control, id=%d, val=%d\n",
 911                          ctrl->id, ctrl->val);
 912                 ret = -EINVAL;
 913         }
 914
 915         vpu_debug_leave();
 916
 917         return ret;
 918 }
 919
 920 static const struct v4l2_ctrl_ops rockchip_vpu_enc_ctrl_ops = {
 921         .s_ctrl = rockchip_vpu_enc_s_ctrl,
 922         .g_volatile_ctrl = rockchip_vpu_enc_g_volatile_ctrl,
 923 };
 924
 925 static int vidioc_cropcap(struct file *file, void *priv,
 926                           struct v4l2_cropcap *cap)
 927 {
 928         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 929         struct v4l2_pix_format_mplane *fmt = &ctx->src_fmt;
 930         int ret = 0;
 931
 932         vpu_debug_enter();
 933
 934         /* Crop only supported on source. */
 935         if (cap->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 936                 ret = -EINVAL;
 937                 goto out;
 938         }
 939
 940         cap->bounds.left = 0;
 941         cap->bounds.top = 0;
 942         cap->bounds.width = fmt->width;
 943         cap->bounds.height = fmt->height;
 944         cap->defrect = cap->bounds;
 945         cap->pixelaspect.numerator = 1;
 946         cap->pixelaspect.denominator = 1;
 947
 948 out:
 949         vpu_debug_leave();
 950
 951         return ret;
 952 }
 953
 954 static int vidioc_g_crop(struct file *file, void *priv, struct v4l2_crop *crop)
 955 {
 956         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 957         int ret = 0;
 958
 959         vpu_debug_enter();
 960
 961         /* Crop only supported on source. */
 962         if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 963                 ret = -EINVAL;
 964                 goto out;
 965         }
 966
 967         crop->c = ctx->src_crop;
 968
 969 out:
 970         vpu_debug_leave();
 971
 972         return ret;
 973 }
 974
 975 static int vidioc_s_crop(struct file *file, void *priv,
 976                          const struct v4l2_crop *crop)
 977 {
 978         struct rockchip_vpu_ctx *ctx = fh_to_ctx(priv);
 979         struct v4l2_pix_format_mplane *fmt = &ctx->src_fmt;
 980         const struct v4l2_rect *rect = &crop->c;
 981         int ret = 0;
 982
 983         vpu_debug_enter();
 984
 985         /* Crop only supported on source. */
 986         if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 987                 ret = -EINVAL;
 988                 goto out;
 989         }
 990
 991         /* Change not allowed if the queue is streaming. */
 992         if (vb2_is_streaming(&ctx->vq_src)) {
 993                 ret = -EBUSY;
 994                 goto out;
 995         }
 996
 997         /* We do not support offsets. */
 998         if (rect->left != 0 || rect->top != 0)
 999                 goto fallback;
1000
1001         /* We can crop only inside right- or bottom-most macroblocks. */
1002         if (round_up(rect->width, MB_DIM) != fmt->width
1003             || round_up(rect->height, MB_DIM) != fmt->height)
1004                 goto fallback;
1005
1006         /* We support widths aligned to 4 pixels and arbitrary heights. */
1007         ctx->src_crop.width = round_up(rect->width, 4);
1008         ctx->src_crop.height = rect->height;
1009
1010         vpu_debug_leave();
1011
1012         return 0;
1013
1014 fallback:
1015         /* Default to full frame for incorrect settings. */
1016         ctx->src_crop.width = fmt->width;
1017         ctx->src_crop.height = fmt->height;
1018
1019 out:
1020         vpu_debug_leave();
1021
1022         return ret;
1023 }
1024
1025 static const struct v4l2_ioctl_ops rockchip_vpu_enc_ioctl_ops = {
1026         .vidioc_querycap = vidioc_querycap,
1027         .vidioc_enum_framesizes = vidioc_enum_framesizes,
1028         .vidioc_enum_fmt_vid_cap_mplane = vidioc_enum_fmt_vid_cap_mplane,
1029         .vidioc_enum_fmt_vid_out_mplane = vidioc_enum_fmt_vid_out_mplane,
1030         .vidioc_g_fmt_vid_cap_mplane = vidioc_g_fmt,
1031         .vidioc_g_fmt_vid_out_mplane = vidioc_g_fmt,
1032         .vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt,
1033         .vidioc_try_fmt_vid_out_mplane = vidioc_try_fmt,
1034         .vidioc_s_fmt_vid_cap_mplane = vidioc_s_fmt,
1035         .vidioc_s_fmt_vid_out_mplane = vidioc_s_fmt,
1036         .vidioc_reqbufs = vidioc_reqbufs,
1037         .vidioc_querybuf = vidioc_querybuf,
1038         .vidioc_qbuf = vidioc_qbuf,
1039         .vidioc_dqbuf = vidioc_dqbuf,
1040         .vidioc_expbuf = vidioc_expbuf,
1041         .vidioc_streamon = vidioc_streamon,
1042         .vidioc_streamoff = vidioc_streamoff,
1043         .vidioc_cropcap = vidioc_cropcap,
1044         .vidioc_g_crop = vidioc_g_crop,
1045         .vidioc_s_crop = vidioc_s_crop,
1046 };
1047
1048 static int rockchip_vpu_queue_setup(struct vb2_queue *vq,
1049                                   const void *parg,
1050                                   unsigned int *buf_count,
1051                                   unsigned int *plane_count,
1052                                   unsigned int psize[], void *allocators[])
1053 {
1054         struct rockchip_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1055         int ret = 0;
1056         int i;
1057
1058         vpu_debug_enter();
1059
1060         switch (vq->type) {
1061         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1062                 *plane_count = ctx->vpu_dst_fmt->num_planes;
1063
1064                 if (*buf_count < 1)
1065                         *buf_count = 1;
1066
1067                 if (*buf_count > VIDEO_MAX_FRAME)
1068                         *buf_count = VIDEO_MAX_FRAME;
1069
1070                 psize[0] = ctx->dst_fmt.plane_fmt[0].sizeimage;
1071                 /* Kernel mapping necessary for bitstream post processing. */
1072                 allocators[0] = ctx->dev->alloc_ctx_vm;
1073                 vpu_debug(0, "capture psize[%d]: %d\n", 0, psize[0]);
1074                 break;
1075
1076         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1077                 *plane_count = ctx->vpu_src_fmt->num_planes;
1078
1079                 if (*buf_count < 1)
1080                         *buf_count = 1;
1081
1082                 if (*buf_count > VIDEO_MAX_FRAME)
1083                         *buf_count = VIDEO_MAX_FRAME;
1084
1085                 for (i = 0; i < ctx->vpu_src_fmt->num_planes; ++i) {
1086                         psize[i] = ctx->src_fmt.plane_fmt[i].sizeimage;
1087                         vpu_debug(0, "output psize[%d]: %d\n", i, psize[i]);
1088                         allocators[i] = ctx->dev->alloc_ctx;
1089                 }
1090                 break;
1091
1092         default:
1093                 vpu_err("invalid queue type: %d\n", vq->type);
1094                 ret = -EINVAL;
1095         }
1096
1097         vpu_debug_leave();
1098
1099         return ret;
1100 }
1101
1102 static int rockchip_vpu_buf_prepare(struct vb2_buffer *vb)
1103 {
1104         struct vb2_queue *vq = vb->vb2_queue;
1105         struct rockchip_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1106         int ret = 0;
1107         int i;
1108
1109         vpu_debug_enter();
1110
1111         switch (vq->type) {
1112         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1113                 vpu_debug(4, "plane size: %ld, dst size: %d\n",
1114                                 vb2_plane_size(vb, 0),
1115                                 ctx->dst_fmt.plane_fmt[0].sizeimage);
1116
1117                 if (vb2_plane_size(vb, 0)
1118                     < ctx->dst_fmt.plane_fmt[0].sizeimage) {
1119                         vpu_err("plane size is too small for capture\n");
1120                         ret = -EINVAL;
1121                 }
1122                 break;
1123
1124         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1125                 for (i = 0; i < ctx->vpu_src_fmt->num_planes; ++i) {
1126                         vpu_debug(4, "plane %d size: %ld, sizeimage: %u\n", i,
1127                                         vb2_plane_size(vb, i),
1128                                         ctx->src_fmt.plane_fmt[i].sizeimage);
1129
1130                         if (vb2_plane_size(vb, i)
1131                             < ctx->src_fmt.plane_fmt[i].sizeimage) {
1132                                 vpu_err("size of plane %d is too small for output\n",
1133                                         i);
1134                                 break;
1135                         }
1136                 }
1137
1138                 if (i != ctx->vpu_src_fmt->num_planes)
1139                         ret = -EINVAL;
1140                 break;
1141
1142         default:
1143                 vpu_err("invalid queue type: %d\n", vq->type);
1144                 ret = -EINVAL;
1145         }
1146
1147         vpu_debug_leave();
1148
1149         return ret;
1150 }
1151
1152 static void rockchip_vpu_buf_finish(struct vb2_buffer *vb)
1153 {
1154         struct vb2_queue *vq = vb->vb2_queue;
1155         struct rockchip_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1156
1157         vpu_debug_enter();
1158
1159         if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE
1160             && vb->state == VB2_BUF_STATE_DONE
1161             && ctx->vpu_dst_fmt->fourcc == V4L2_PIX_FMT_VP8) {
1162                 struct rockchip_vpu_buf *buf;
1163
1164                 buf = vb_to_buf(vb);
1165                 rk3288_vpu_vp8e_assemble_bitstream(ctx, buf);
1166         }
1167
1168         vpu_debug_leave();
1169 }
1170
1171 static int rockchip_vpu_start_streaming(struct vb2_queue *q, unsigned int count)
1172 {
1173         int ret = 0;
1174         struct rockchip_vpu_ctx *ctx = fh_to_ctx(q->drv_priv);
1175         struct rockchip_vpu_dev *dev = ctx->dev;
1176         bool ready = false;
1177
1178         vpu_debug_enter();
1179
1180         if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
1181                 ret = rockchip_vpu_init(ctx);
1182                 if (ret < 0) {
1183                         vpu_err("rockchip_vpu_init failed\n");
1184                         return ret;
1185                 }
1186
1187                 ready = vb2_is_streaming(&ctx->vq_src);
1188         } else if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
1189                 ready = vb2_is_streaming(&ctx->vq_dst);
1190         }
1191
1192         if (ready)
1193                 rockchip_vpu_try_context(dev, ctx);
1194
1195         vpu_debug_leave();
1196
1197         return 0;
1198 }
1199
1200 static void rockchip_vpu_stop_streaming(struct vb2_queue *q)
1201 {
1202         unsigned long flags;
1203         struct rockchip_vpu_ctx *ctx = fh_to_ctx(q->drv_priv);
1204         struct rockchip_vpu_dev *dev = ctx->dev;
1205         struct rockchip_vpu_buf *b;
1206         LIST_HEAD(queue);
1207         int i;
1208
1209         vpu_debug_enter();
1210
1211         spin_lock_irqsave(&dev->irqlock, flags);
1212
1213         list_del_init(&ctx->list);
1214
1215         switch (q->type) {
1216         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1217                 list_splice_init(&ctx->dst_queue, &queue);
1218                 break;
1219
1220         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1221                 list_splice_init(&ctx->src_queue, &queue);
1222                 break;
1223
1224         default:
1225                 break;
1226         }
1227
1228         spin_unlock_irqrestore(&dev->irqlock, flags);
1229
1230         wait_event(dev->run_wq, dev->current_ctx != ctx);
1231
1232         while (!list_empty(&queue)) {
1233                 b = list_first_entry(&queue, struct rockchip_vpu_buf, list);
1234                 for (i = 0; i < b->vb.vb2_buf.num_planes; i++)
1235                         vb2_set_plane_payload(&b->vb.vb2_buf, i, 0);
1236                 vb2_buffer_done(&b->vb.vb2_buf, VB2_BUF_STATE_ERROR);
1237                 list_del(&b->list);
1238         }
1239
1240         if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
1241                 rockchip_vpu_deinit(ctx);
1242
1243         vpu_debug_leave();
1244 }
1245
1246 static void rockchip_vpu_buf_queue(struct vb2_buffer *vb)
1247 {
1248         struct vb2_queue *vq = vb->vb2_queue;
1249         struct rockchip_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1250         struct rockchip_vpu_dev *dev = ctx->dev;
1251         struct rockchip_vpu_buf *vpu_buf;
1252         unsigned long flags;
1253
1254         vpu_debug_enter();
1255
1256         switch (vq->type) {
1257         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1258                 vpu_buf = vb_to_buf(vb);
1259
1260                 /* Mark destination as available for use by VPU */
1261                 spin_lock_irqsave(&dev->irqlock, flags);
1262
1263                 list_add_tail(&vpu_buf->list, &ctx->dst_queue);
1264
1265                 spin_unlock_irqrestore(&dev->irqlock, flags);
1266                 break;
1267
1268         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1269                 vpu_buf = vb_to_buf(vb);
1270
1271                 spin_lock_irqsave(&dev->irqlock, flags);
1272
1273                 list_add_tail(&vpu_buf->list, &ctx->src_queue);
1274
1275                 spin_unlock_irqrestore(&dev->irqlock, flags);
1276                 break;
1277
1278         default:
1279                 vpu_err("unsupported buffer type (%d)\n", vq->type);
1280         }
1281
1282         if (vb2_is_streaming(&ctx->vq_src) && vb2_is_streaming(&ctx->vq_dst))
1283                 rockchip_vpu_try_context(dev, ctx);
1284
1285         vpu_debug_leave();
1286 }
1287
1288 static struct vb2_ops rockchip_vpu_enc_qops = {
1289         .queue_setup = rockchip_vpu_queue_setup,
1290         .wait_prepare = vb2_ops_wait_prepare,
1291         .wait_finish = vb2_ops_wait_finish,
1292         .buf_prepare = rockchip_vpu_buf_prepare,
1293         .buf_finish = rockchip_vpu_buf_finish,
1294         .start_streaming = rockchip_vpu_start_streaming,
1295         .stop_streaming = rockchip_vpu_stop_streaming,
1296         .buf_queue = rockchip_vpu_buf_queue,
1297 };
1298
1299 struct vb2_ops *get_enc_queue_ops(void)
1300 {
1301         return &rockchip_vpu_enc_qops;
1302 }
1303
1304 const struct v4l2_ioctl_ops *get_enc_v4l2_ioctl_ops(void)
1305 {
1306         return &rockchip_vpu_enc_ioctl_ops;
1307 }
1308
1309 static void rockchip_vpu_enc_prepare_run(struct rockchip_vpu_ctx *ctx)
1310 {
1311         struct vb2_v4l2_buffer *vb2_src = to_vb2_v4l2_buffer(&ctx->run.src->vb.vb2_buf);
1312         unsigned config_store = vb2_src->config_store;
1313
1314         v4l2_ctrl_apply_store(&ctx->ctrl_handler, config_store);
1315
1316         memcpy(ctx->run.dst->vp8e.header,
1317                 get_ctrl_ptr(ctx, ROCKCHIP_VPU_ENC_CTRL_HEADER),
1318                 ROCKCHIP_HEADER_SIZE);
1319         ctx->run.vp8e.reg_params = get_ctrl_ptr(ctx,
1320                 ROCKCHIP_VPU_ENC_CTRL_REG_PARAMS);
1321         memcpy(ctx->run.priv_src.cpu,
1322                 get_ctrl_ptr(ctx, ROCKCHIP_VPU_ENC_CTRL_HW_PARAMS),
1323                 ROCKCHIP_HW_PARAMS_SIZE);
1324 }
1325
1326 static const struct rockchip_vpu_run_ops rockchip_vpu_enc_run_ops = {
1327         .prepare_run = rockchip_vpu_enc_prepare_run,
1328 };
1329
1330 int rockchip_vpu_enc_init(struct rockchip_vpu_ctx *ctx)
1331 {
1332         struct rockchip_vpu_dev *vpu = ctx->dev;
1333         int ret;
1334
1335         ctx->vpu_src_fmt = find_format(DEF_SRC_FMT_ENC, false);
1336         ctx->vpu_dst_fmt = find_format(DEF_DST_FMT_ENC, true);
1337
1338         ret = rockchip_vpu_aux_buf_alloc(vpu, &ctx->run.priv_src,
1339                                         ROCKCHIP_HW_PARAMS_SIZE);
1340         if (ret) {
1341                 vpu_err("Failed to allocate private source buffer.\n");
1342                 return ret;
1343         }
1344
1345
1346         ret = rockchip_vpu_aux_buf_alloc(vpu, &ctx->run.priv_dst,
1347                                         ROCKCHIP_RET_PARAMS_SIZE);
1348         if (ret) {
1349                 vpu_err("Failed to allocate private destination buffer.\n");
1350                 goto err_priv_src;
1351         }
1352
1353         ret = rockchip_vpu_ctrls_setup(ctx, &rockchip_vpu_enc_ctrl_ops,
1354                                         controls, ARRAY_SIZE(controls),
1355                                         rockchip_vpu_enc_get_menu);
1356         if (ret) {
1357                 vpu_err("Failed to set up controls\n");
1358                 goto err_priv_dst;
1359         }
1360
1361         ctx->run_ops = &rockchip_vpu_enc_run_ops;
1362
1363         return 0;
1364
1365 err_priv_dst:
1366         rockchip_vpu_aux_buf_free(vpu, &ctx->run.priv_dst);
1367 err_priv_src:
1368         rockchip_vpu_aux_buf_free(vpu, &ctx->run.priv_src);
1369
1370         return ret;
1371 }
1372
1373 void rockchip_vpu_enc_exit(struct rockchip_vpu_ctx *ctx)
1374 {
1375         struct rockchip_vpu_dev *vpu = ctx->dev;
1376
1377         rockchip_vpu_ctrls_delete(ctx);
1378
1379         rockchip_vpu_aux_buf_free(vpu, &ctx->run.priv_dst);
1380         rockchip_vpu_aux_buf_free(vpu, &ctx->run.priv_src);
1381 };
1382
1383 /*
1384  * WAR for encoder state corruption after decoding
1385  */
1386
1387 static const struct rockchip_vpu_run_ops dummy_encode_run_ops = {
1388         /* No ops needed for dummy encoding. */
1389 };
1390
1391 #define DUMMY_W         64
1392 #define DUMMY_H         64
1393 #define DUMMY_SRC_FMT   V4L2_PIX_FMT_YUYV
1394 #define DUMMY_DST_FMT   V4L2_PIX_FMT_VP8
1395 #define DUMMY_DST_SIZE  (32 * 1024)
1396
1397 int rockchip_vpu_enc_init_dummy_ctx(struct rockchip_vpu_dev *dev)
1398 {
1399         struct rockchip_vpu_ctx *ctx;
1400         int ret;
1401         int i;
1402
1403         ctx = devm_kzalloc(dev->dev, sizeof(*ctx), GFP_KERNEL);
1404         if (!ctx)
1405                 return -ENOMEM;
1406
1407         ctx->dev = dev;
1408
1409         ctx->vpu_src_fmt = find_format(DUMMY_SRC_FMT, false);
1410         ctx->src_fmt.width = DUMMY_W;
1411         ctx->src_fmt.height = DUMMY_H;
1412         ctx->src_fmt.pixelformat = ctx->vpu_src_fmt->fourcc;
1413         ctx->src_fmt.num_planes = ctx->vpu_src_fmt->num_planes;
1414
1415         calculate_plane_sizes(ctx->vpu_src_fmt, ctx->src_fmt.width,
1416                                 ctx->src_fmt.height, &ctx->src_fmt);
1417
1418         ctx->vpu_dst_fmt = find_format(DUMMY_DST_FMT, true);
1419         ctx->dst_fmt.width = ctx->src_fmt.width;
1420         ctx->dst_fmt.height = ctx->src_fmt.height;
1421         ctx->dst_fmt.pixelformat = ctx->vpu_dst_fmt->fourcc;
1422         ctx->dst_fmt.plane_fmt[0].sizeimage = DUMMY_DST_SIZE;
1423         ctx->dst_fmt.plane_fmt[0].bytesperline = 0;
1424         ctx->dst_fmt.num_planes = 1;
1425
1426         INIT_LIST_HEAD(&ctx->src_queue);
1427
1428         ctx->src_crop.left = 0;
1429         ctx->src_crop.top = 0;
1430         ctx->src_crop.width = ctx->src_fmt.width;
1431         ctx->src_crop.left = ctx->src_fmt.height;
1432
1433         INIT_LIST_HEAD(&ctx->dst_queue);
1434         INIT_LIST_HEAD(&ctx->list);
1435
1436         ctx->run.vp8e.reg_params =
1437                 rk3288_vpu_vp8e_get_dummy_params();
1438         ctx->run_ops = &dummy_encode_run_ops;
1439
1440         ctx->run.dst = devm_kzalloc(dev->dev, sizeof(*ctx->run.dst),
1441                                         GFP_KERNEL);
1442         if (!ctx->run.dst)
1443                 return -ENOMEM;
1444
1445         ret = rockchip_vpu_aux_buf_alloc(dev, &ctx->run.priv_src,
1446                                         ROCKCHIP_HW_PARAMS_SIZE);
1447         if (ret)
1448                 return ret;
1449
1450         ret = rockchip_vpu_aux_buf_alloc(dev, &ctx->run.priv_dst,
1451                                         ROCKCHIP_RET_PARAMS_SIZE);
1452         if (ret)
1453                 goto err_free_priv_src;
1454
1455         for (i = 0; i < ctx->src_fmt.num_planes; ++i) {
1456                 ret = rockchip_vpu_aux_buf_alloc(dev, &dev->dummy_encode_src[i],
1457                                         ctx->src_fmt.plane_fmt[i].sizeimage);
1458                 if (ret)
1459                         goto err_free_src;
1460
1461                 memset(dev->dummy_encode_src[i].cpu, 0,
1462                         dev->dummy_encode_src[i].size);
1463         }
1464
1465         ret = rockchip_vpu_aux_buf_alloc(dev, &dev->dummy_encode_dst,
1466                                         ctx->dst_fmt.plane_fmt[0].sizeimage);
1467         if (ret)
1468                 goto err_free_src;
1469
1470         memset(dev->dummy_encode_dst.cpu, 0, dev->dummy_encode_dst.size);
1471
1472         ret = rockchip_vpu_init(ctx);
1473         if (ret)
1474                 goto err_free_dst;
1475
1476         dev->dummy_encode_ctx = ctx;
1477
1478         return 0;
1479
1480 err_free_dst:
1481         rockchip_vpu_aux_buf_free(dev, &dev->dummy_encode_dst);
1482 err_free_src:
1483         for (i = 0; i < ctx->src_fmt.num_planes; ++i)
1484                 if (dev->dummy_encode_src[i].cpu)
1485                         rockchip_vpu_aux_buf_free(dev, &dev->dummy_encode_src[i]);
1486         rockchip_vpu_aux_buf_free(dev, &ctx->run.priv_dst);
1487 err_free_priv_src:
1488         rockchip_vpu_aux_buf_free(dev, &ctx->run.priv_src);
1489
1490         return ret;
1491 }
1492
1493 void rockchip_vpu_enc_free_dummy_ctx(struct rockchip_vpu_dev *dev)
1494 {
1495         struct rockchip_vpu_ctx *ctx = dev->dummy_encode_ctx;
1496         int i;
1497
1498         rockchip_vpu_deinit(ctx);
1499
1500         for (i = 0; i < ctx->src_fmt.num_planes; ++i)
1501                 rockchip_vpu_aux_buf_free(dev, &dev->dummy_encode_src[i]);
1502         rockchip_vpu_aux_buf_free(dev, &dev->dummy_encode_dst);
1503         rockchip_vpu_aux_buf_free(dev, &ctx->run.priv_src);
1504         rockchip_vpu_aux_buf_free(dev, &ctx->run.priv_dst);
1505 }