drivers/media/platform/rk3288-vpu/rk3288_vpu_enc.c

   1 /*
   2  * Rockchip RK3288 VPU codec driver
   3  *
   4  * Copyright (C) 2014 Rockchip Electronics Co., Ltd.
   5  *      Alpha Lin <Alpha.Lin@rock-chips.com>
   6  *      Jeffy Chen <jeffy.chen@rock-chips.com>
   7  *
   8  * Copyright (C) 2014 Google, Inc.
   9  *      Tomasz Figa <tfiga@chromium.org>
  10  *
  11  * Based on s5p-mfc driver by Samsung Electronics Co., Ltd.
  12  *
  13  * Copyright (C) 2010-2011 Samsung Electronics Co., Ltd.
  14  *
  15  * This software is licensed under the terms of the GNU General Public
  16  * License version 2, as published by the Free Software Foundation, and
  17  * may be copied, distributed, and modified under those terms.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include "rk3288_vpu_common.h"
  26
  27 #include <linux/clk.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/io.h>
  30 #include <linux/module.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/sched.h>
  33 #include <linux/version.h>
  34 #include <linux/videodev2.h>
  35 #include <media/v4l2-event.h>
  36 #include <linux/workqueue.h>
  37 #include <media/v4l2-ctrls.h>
  38 #include <media/videobuf2-core.h>
  39 #include <media/videobuf2-dma-sg.h>
  40
  41 #include "rk3288_vpu_enc.h"
  42 #include "rk3288_vpu_hw.h"
  43
  44 #define DEF_SRC_FMT_ENC                         V4L2_PIX_FMT_NV12
  45 #define DEF_DST_FMT_ENC                         V4L2_PIX_FMT_VP8
  46
  47 #define RK3288_ENC_MIN_WIDTH                    96U
  48 #define RK3288_ENC_MAX_WIDTH                    1920U
  49 #define RK3288_ENC_MIN_HEIGHT                   96U
  50 #define RK3288_ENC_MAX_HEIGHT                   1088U
  51
  52 #define V4L2_CID_PRIVATE_RK3288_HEADER          (V4L2_CID_CUSTOM_BASE + 0)
  53 #define V4L2_CID_PRIVATE_RK3288_REG_PARAMS      (V4L2_CID_CUSTOM_BASE + 1)
  54 #define V4L2_CID_PRIVATE_RK3288_HW_PARAMS       (V4L2_CID_CUSTOM_BASE + 2)
  55 #define V4L2_CID_PRIVATE_RK3288_RET_PARAMS      (V4L2_CID_CUSTOM_BASE + 3)
  56
  57 static struct rk3288_vpu_fmt formats[] = {
  58         /* Source formats. */
  59         {
  60                 .name = "4:2:0 3 planes Y/Cb/Cr",
  61                 .fourcc = V4L2_PIX_FMT_YUV420M,
  62                 .codec_mode = RK_VPU_CODEC_NONE,
  63                 .num_planes = 3,
  64                 .depth = { 8, 4, 4 },
  65                 .enc_fmt = RK3288_VPU_ENC_FMT_YUV420P,
  66         },
  67         {
  68                 .name = "4:2:0 2 plane Y/CbCr",
  69                 .fourcc = V4L2_PIX_FMT_NV12M,
  70                 .codec_mode = RK_VPU_CODEC_NONE,
  71                 .num_planes = 2,
  72                 .depth = { 8, 8 },
  73                 .enc_fmt = RK3288_VPU_ENC_FMT_YUV420SP,
  74         },
  75         {
  76                 .name = "4:2:2 1 plane YUYV",
  77                 .fourcc = V4L2_PIX_FMT_YUYV,
  78                 .codec_mode = RK_VPU_CODEC_NONE,
  79                 .num_planes = 1,
  80                 .depth = { 16 },
  81                 .enc_fmt = RK3288_VPU_ENC_FMT_YUYV422,
  82         },
  83         {
  84                 .name = "4:2:2 1 plane UYVY",
  85                 .fourcc = V4L2_PIX_FMT_UYVY,
  86                 .codec_mode = RK_VPU_CODEC_NONE,
  87                 .num_planes = 1,
  88                 .depth = { 16 },
  89                 .enc_fmt = RK3288_VPU_ENC_FMT_UYVY422,
  90         },
  91         /* Destination formats. */
  92         {
  93                 .name = "VP8 Encoded Stream",
  94                 .fourcc = V4L2_PIX_FMT_VP8,
  95                 .codec_mode = RK_VPU_CODEC_VP8E,
  96                 .num_planes = 1,
  97         },
  98 };
  99
 100 static struct rk3288_vpu_fmt *find_format(struct v4l2_format *f, bool bitstream)
 101 {
 102         unsigned int i;
 103
 104         vpu_debug_enter();
 105
 106         for (i = 0; i < ARRAY_SIZE(formats); i++) {
 107                 if (formats[i].fourcc != f->fmt.pix_mp.pixelformat)
 108                         continue;
 109                 if (bitstream && formats[i].codec_mode != RK_VPU_CODEC_NONE)
 110                         return &formats[i];
 111                 if (!bitstream && formats[i].codec_mode == RK_VPU_CODEC_NONE)
 112                         return &formats[i];
 113         }
 114
 115         return NULL;
 116 }
 117
 118 /*
 119  * Indices of controls that need to be accessed directly, i.e. through
 120  * p_cur.p pointer of their v4l2_ctrl structs.
 121  */
 122 enum {
 123         RK3288_VPU_ENC_CTRL_HEADER,
 124         RK3288_VPU_ENC_CTRL_REG_PARAMS,
 125         RK3288_VPU_ENC_CTRL_HW_PARAMS,
 126         RK3288_VPU_ENC_CTRL_RET_PARAMS,
 127 };
 128
 129 static struct rk3288_vpu_control controls[] = {
 130         /* Private, per-frame controls. */
 131         [RK3288_VPU_ENC_CTRL_HEADER] = {
 132                 .id = V4L2_CID_PRIVATE_RK3288_HEADER,
 133                 .type = V4L2_CTRL_TYPE_PRIVATE,
 134                 .name = "Rk3288 Private Header",
 135                 .elem_size = RK3288_HEADER_SIZE,
 136                 .max_stores = VIDEO_MAX_FRAME,
 137                 .can_store = true,
 138         },
 139         [RK3288_VPU_ENC_CTRL_REG_PARAMS] = {
 140                 .id = V4L2_CID_PRIVATE_RK3288_REG_PARAMS,
 141                 .type = V4L2_CTRL_TYPE_PRIVATE,
 142                 .name = "Rk3288 Private Reg Params",
 143                 .elem_size = sizeof(struct rk3288_vp8e_reg_params),
 144                 .max_stores = VIDEO_MAX_FRAME,
 145                 .can_store = true,
 146         },
 147         [RK3288_VPU_ENC_CTRL_HW_PARAMS] = {
 148                 .id = V4L2_CID_PRIVATE_RK3288_HW_PARAMS,
 149                 .type = V4L2_CTRL_TYPE_PRIVATE,
 150                 .name = "Rk3288 Private Hw Params",
 151                 .elem_size = RK3288_HW_PARAMS_SIZE,
 152                 .max_stores = VIDEO_MAX_FRAME,
 153                 .can_store = true,
 154         },
 155         [RK3288_VPU_ENC_CTRL_RET_PARAMS] = {
 156                 .id = V4L2_CID_PRIVATE_RK3288_RET_PARAMS,
 157                 .type = V4L2_CTRL_TYPE_PRIVATE,
 158                 .name = "Rk3288 Private Ret Params",
 159                 .is_volatile = true,
 160                 .is_read_only = true,
 161                 .max_stores = VIDEO_MAX_FRAME,
 162                 .elem_size = RK3288_RET_PARAMS_SIZE,
 163         },
 164         /* Generic controls. (currently ignored) */
 165         {
 166                 .id = V4L2_CID_MPEG_VIDEO_GOP_SIZE,
 167                 .type = V4L2_CTRL_TYPE_INTEGER,
 168                 .minimum = 1,
 169                 .maximum = 150,
 170                 .step = 1,
 171                 .default_value = 30,
 172         },
 173         {
 174                 .id = V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE,
 175                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 176                 .minimum = 0,
 177                 .maximum = 1,
 178                 .step = 1,
 179                 .default_value = 0,
 180         },
 181         {
 182                 .id = V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE,
 183                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 184                 .minimum = 0,
 185                 .maximum = 1,
 186                 .step = 1,
 187                 .default_value = 0,
 188         },
 189         {
 190                 .id = V4L2_CID_MPEG_VIDEO_BITRATE,
 191                 .type = V4L2_CTRL_TYPE_INTEGER,
 192                 .minimum = 10000,
 193                 .maximum = 288000000,
 194                 .step = 1,
 195                 .default_value = 2097152,
 196         },
 197         {
 198                 .id = V4L2_CID_MPEG_VIDEO_H264_PROFILE,
 199                 .type = V4L2_CTRL_TYPE_MENU,
 200                 .minimum = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
 201                 .maximum = V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH,
 202                 .default_value = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
 203                 .menu_skip_mask = ~(
 204                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
 205                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
 206                                 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)
 207                         ),
 208         },
 209         {
 210                 .id = V4L2_CID_MPEG_VIDEO_H264_LEVEL,
 211                 .type = V4L2_CTRL_TYPE_MENU,
 212                 .minimum = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
 213                 .maximum = V4L2_MPEG_VIDEO_H264_LEVEL_4_1,
 214                 .default_value = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
 215         },
 216         {
 217                 .id = V4L2_CID_MPEG_VIDEO_H264_MAX_QP,
 218                 .type = V4L2_CTRL_TYPE_INTEGER,
 219                 .minimum = 0,
 220                 .maximum = 51,
 221                 .step = 1,
 222                 .default_value = 30,
 223         },
 224         {
 225                 .id = V4L2_CID_MPEG_VIDEO_H264_MIN_QP,
 226                 .type = V4L2_CTRL_TYPE_INTEGER,
 227                 .minimum = 0,
 228                 .maximum = 51,
 229                 .step = 1,
 230                 .default_value = 18,
 231         },
 232         {
 233                 .id = V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM,
 234                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 235                 .minimum = 0,
 236                 .maximum = 1,
 237                 .step = 1,
 238                 .default_value = 0,
 239         },
 240         {
 241                 .id = V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE,
 242                 .type = V4L2_CTRL_TYPE_INTEGER,
 243                 .minimum = 0,
 244                 .maximum = 288000,
 245                 .step = 1,
 246                 .default_value = 30000,
 247         },
 248         {
 249                 .id = V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING,
 250                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 251                 .minimum = 0,
 252                 .maximum = 1,
 253                 .step = 1,
 254                 .default_value = 0,
 255         },
 256         {
 257                 .id = V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE,
 258                 .type = V4L2_CTRL_TYPE_MENU,
 259                 .name = "Force frame type",
 260                 .minimum = V4L2_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE_DISABLED,
 261                 .maximum = V4L2_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE_NOT_CODED,
 262                 .default_value =
 263                         V4L2_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE_DISABLED,
 264                 .menu_skip_mask = 0,
 265         },
 266         /*
 267          * This hardware does not support features provided by controls
 268          * below, but they are required for compatibility with certain
 269          * userspace software.
 270          */
 271         {
 272                 .id = V4L2_CID_MPEG_MFC51_VIDEO_RC_REACTION_COEFF,
 273                 .type = V4L2_CTRL_TYPE_INTEGER,
 274                 .name = "Rate Control Reaction Coeff.",
 275                 .minimum = 1,
 276                 .maximum = (1 << 16) - 1,
 277                 .step = 1,
 278                 .default_value = 1,
 279         },
 280         {
 281                 .id = V4L2_CID_MPEG_VIDEO_HEADER_MODE,
 282                 .type = V4L2_CTRL_TYPE_MENU,
 283                 .minimum = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
 284                 .maximum = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
 285                 .default_value = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
 286                 .menu_skip_mask = 0,
 287         },
 288         {
 289                 .id = V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT,
 290                 .type = V4L2_CTRL_TYPE_BOOLEAN,
 291                 .name = "Fixed Target Bit Enable",
 292                 .minimum = 0,
 293                 .maximum = 1,
 294                 .default_value = 0,
 295                 .step = 1,
 296                 .menu_skip_mask = 0,
 297         },
 298         {
 299                 .id = V4L2_CID_MPEG_VIDEO_B_FRAMES,
 300                 .type = V4L2_CTRL_TYPE_INTEGER,
 301                 .minimum = 0,
 302                 .maximum = 2,
 303                 .step = 1,
 304                 .default_value = 0,
 305         },
 306         {
 307                 .id = V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP,
 308                 .type = V4L2_CTRL_TYPE_INTEGER,
 309                 .minimum = 0,
 310                 .maximum = 51,
 311                 .step = 1,
 312                 .default_value = 1,
 313         },
 314 };
 315
 316 static inline const void *get_ctrl_ptr(struct rk3288_vpu_ctx *ctx, unsigned id)
 317 {
 318         struct v4l2_ctrl *ctrl = ctx->ctrls[id];
 319
 320         return ctrl->p_cur.p;
 321 }
 322
 323 static const char *const *rk3288_vpu_enc_get_menu(u32 id)
 324 {
 325         static const char *const vpu_video_frame_skip[] = {
 326                 "Disabled",
 327                 "Level Limit",
 328                 "VBV/CPB Limit",
 329                 NULL,
 330         };
 331         static const char *const vpu_video_force_frame[] = {
 332                 "Disabled",
 333                 "I Frame",
 334                 "Not Coded",
 335                 NULL,
 336         };
 337
 338         switch (id) {
 339         case V4L2_CID_MPEG_MFC51_VIDEO_FRAME_SKIP_MODE:
 340                 return vpu_video_frame_skip;
 341         case V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE:
 342                 return vpu_video_force_frame;
 343         }
 344
 345         return NULL;
 346 }
 347
 348 static int vidioc_querycap(struct file *file, void *priv,
 349                            struct v4l2_capability *cap)
 350 {
 351         struct rk3288_vpu_dev *dev = video_drvdata(file);
 352
 353         vpu_debug_enter();
 354
 355         strlcpy(cap->driver, RK3288_VPU_ENC_NAME, sizeof(cap->driver));
 356         strlcpy(cap->card, dev->pdev->name, sizeof(cap->card));
 357         strlcpy(cap->bus_info, "platform:" RK3288_VPU_NAME,
 358                 sizeof(cap->bus_info));
 359
 360         /*
 361          * This is only a mem-to-mem video device. The capture and output
 362          * device capability flags are left only for backward compatibility
 363          * and are scheduled for removal.
 364          */
 365         cap->capabilities = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING |
 366             V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_OUTPUT_MPLANE;
 367
 368         vpu_debug_leave();
 369
 370         return 0;
 371 }
 372
 373 static int vidioc_enum_fmt(struct v4l2_fmtdesc *f, bool out)
 374 {
 375         struct rk3288_vpu_fmt *fmt;
 376         int i, j = 0;
 377
 378         vpu_debug_enter();
 379
 380         for (i = 0; i < ARRAY_SIZE(formats); ++i) {
 381                 if (out && formats[i].codec_mode != RK_VPU_CODEC_NONE)
 382                         continue;
 383                 else if (!out && formats[i].codec_mode == RK_VPU_CODEC_NONE)
 384                         continue;
 385
 386                 if (j == f->index) {
 387                         fmt = &formats[i];
 388                         strlcpy(f->description, fmt->name,
 389                                 sizeof(f->description));
 390                         f->pixelformat = fmt->fourcc;
 391
 392                         f->flags = 0;
 393                         if (formats[i].codec_mode != RK_VPU_CODEC_NONE)
 394                                 f->flags |= V4L2_FMT_FLAG_COMPRESSED;
 395
 396                         vpu_debug_leave();
 397
 398                         return 0;
 399                 }
 400
 401                 ++j;
 402         }
 403
 404         vpu_debug_leave();
 405
 406         return -EINVAL;
 407 }
 408
 409 static int vidioc_enum_fmt_vid_cap_mplane(struct file *file, void *priv,
 410                                           struct v4l2_fmtdesc *f)
 411 {
 412         return vidioc_enum_fmt(f, false);
 413 }
 414
 415 static int vidioc_enum_fmt_vid_out_mplane(struct file *file, void *priv,
 416                                           struct v4l2_fmtdesc *f)
 417 {
 418         return vidioc_enum_fmt(f, true);
 419 }
 420
 421 static int vidioc_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
 422 {
 423         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 424
 425         vpu_debug_enter();
 426
 427         vpu_debug(4, "f->type = %d\n", f->type);
 428
 429         switch (f->type) {
 430         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 431                 f->fmt.pix_mp = ctx->dst_fmt;
 432                 break;
 433
 434         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 435                 f->fmt.pix_mp = ctx->src_fmt;
 436                 break;
 437
 438         default:
 439                 vpu_err("invalid buf type\n");
 440                 return -EINVAL;
 441         }
 442
 443         vpu_debug_leave();
 444
 445         return 0;
 446 }
 447
 448 static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f)
 449 {
 450         struct rk3288_vpu_fmt *fmt;
 451         struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
 452         char str[5];
 453
 454         vpu_debug_enter();
 455
 456         switch (f->type) {
 457         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 458                 vpu_debug(4, "%s\n", fmt2str(f->fmt.pix_mp.pixelformat, str));
 459
 460                 fmt = find_format(f, true);
 461                 if (!fmt) {
 462                         vpu_err("failed to try capture format\n");
 463                         return -EINVAL;
 464                 }
 465
 466                 if (pix_fmt_mp->plane_fmt[0].sizeimage == 0) {
 467                         vpu_err("must be set encoding output size\n");
 468                         return -EINVAL;
 469                 }
 470
 471                 pix_fmt_mp->plane_fmt[0].bytesperline = 0;
 472                 break;
 473
 474         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 475                 vpu_debug(4, "%s\n", fmt2str(f->fmt.pix_mp.pixelformat, str));
 476
 477                 fmt = find_format(f, false);
 478                 if (!fmt) {
 479                         vpu_err("failed to try output format\n");
 480                         return -EINVAL;
 481                 }
 482
 483                 if (fmt->num_planes != pix_fmt_mp->num_planes) {
 484                         vpu_err("plane number mismatches on output format\n");
 485                         return -EINVAL;
 486                 }
 487
 488                 /* Limit to hardware min/max. */
 489                 pix_fmt_mp->width = clamp(pix_fmt_mp->width,
 490                                 RK3288_ENC_MIN_WIDTH, RK3288_ENC_MAX_WIDTH);
 491                 pix_fmt_mp->height = clamp(pix_fmt_mp->height,
 492                                 RK3288_ENC_MIN_HEIGHT, RK3288_ENC_MAX_HEIGHT);
 493                 /* Round up to macroblocks. */
 494                 pix_fmt_mp->width = round_up(pix_fmt_mp->width, MB_DIM);
 495                 pix_fmt_mp->height = round_up(pix_fmt_mp->height, MB_DIM);
 496                 break;
 497
 498         default:
 499                 vpu_err("invalid buf type\n");
 500                 return -EINVAL;
 501         }
 502
 503         vpu_debug_leave();
 504
 505         return 0;
 506 }
 507
 508 static int vidioc_s_fmt(struct file *file, void *priv, struct v4l2_format *f)
 509 {
 510         struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
 511         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 512         unsigned int mb_width, mb_height;
 513         struct rk3288_vpu_fmt *fmt;
 514         int ret = 0;
 515         int i;
 516
 517         vpu_debug_enter();
 518
 519         /* Change not allowed if any queue is streaming. */
 520         if (vb2_is_streaming(&ctx->vq_src) || vb2_is_streaming(&ctx->vq_dst)) {
 521                 ret = -EBUSY;
 522                 goto out;
 523         }
 524
 525         switch (f->type) {
 526         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 527                 /*
 528                  * Pixel format change is not allowed when the other queue has
 529                  * buffers allocated.
 530                  */
 531                 if (vb2_is_busy(&ctx->vq_src)
 532                     && pix_fmt_mp->pixelformat != ctx->dst_fmt.pixelformat) {
 533                         ret = -EBUSY;
 534                         goto out;
 535                 }
 536
 537                 ret = vidioc_try_fmt(file, priv, f);
 538                 if (ret)
 539                         goto out;
 540
 541                 ctx->vpu_dst_fmt = find_format(f, true);
 542                 ctx->dst_fmt = *pix_fmt_mp;
 543                 break;
 544
 545         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 546                 /*
 547                  * Pixel format change is not allowed when the other queue has
 548                  * buffers allocated.
 549                  */
 550                 if (vb2_is_busy(&ctx->vq_dst)
 551                     && pix_fmt_mp->pixelformat != ctx->src_fmt.pixelformat) {
 552                         ret = -EBUSY;
 553                         goto out;
 554                 }
 555
 556                 ret = vidioc_try_fmt(file, priv, f);
 557                 if (ret)
 558                         goto out;
 559
 560                 fmt = find_format(f, false);
 561                 ctx->vpu_src_fmt = fmt;
 562
 563                 mb_width = MB_WIDTH(pix_fmt_mp->width);
 564                 mb_height = MB_HEIGHT(pix_fmt_mp->height);
 565
 566                 vpu_debug(0, "OUTPUT codec mode: %d\n", fmt->codec_mode);
 567                 vpu_debug(0, "fmt - w: %d, h: %d, mb - w: %d, h: %d\n",
 568                           pix_fmt_mp->width, pix_fmt_mp->height,
 569                           mb_width, mb_height);
 570
 571                 for (i = 0; i < fmt->num_planes; ++i) {
 572                         pix_fmt_mp->plane_fmt[i].bytesperline =
 573                                 mb_width * MB_DIM * fmt->depth[i] / 8;
 574                         pix_fmt_mp->plane_fmt[i].sizeimage =
 575                                 pix_fmt_mp->plane_fmt[i].bytesperline
 576                                 * mb_height * MB_DIM;
 577                         /*
 578                          * All of multiplanar formats we support have chroma
 579                          * planes subsampled by 2.
 580                          */
 581                         if (i != 0)
 582                                 pix_fmt_mp->plane_fmt[i].sizeimage /= 2;
 583                 }
 584
 585                 /* Reset crop rectangle. */
 586                 ctx->src_crop.width = pix_fmt_mp->width;
 587                 ctx->src_crop.height = pix_fmt_mp->height;
 588
 589                 ctx->src_fmt = *pix_fmt_mp;
 590                 break;
 591
 592         default:
 593                 vpu_err("invalid buf type\n");
 594                 return -EINVAL;
 595         }
 596
 597 out:
 598         vpu_debug_leave();
 599
 600         return ret;
 601 }
 602
 603 static int vidioc_reqbufs(struct file *file, void *priv,
 604                           struct v4l2_requestbuffers *reqbufs)
 605 {
 606         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 607         int ret;
 608
 609         vpu_debug_enter();
 610
 611         switch (reqbufs->type) {
 612         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 613                 vpu_debug(4, "\n");
 614
 615                 ret = vb2_reqbufs(&ctx->vq_dst, reqbufs);
 616                 if (ret != 0) {
 617                         vpu_err("error in vb2_reqbufs() for E(D)\n");
 618                         goto out;
 619                 }
 620                 break;
 621
 622         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 623                 vpu_debug(4, "memory type %d\n", reqbufs->memory);
 624
 625                 ret = vb2_reqbufs(&ctx->vq_src, reqbufs);
 626                 if (ret != 0) {
 627                         vpu_err("error in vb2_reqbufs() for E(S)\n");
 628                         goto out;
 629                 }
 630                 break;
 631
 632         default:
 633                 vpu_err("invalid buf type\n");
 634                 ret = -EINVAL;
 635                 goto out;
 636         }
 637
 638 out:
 639         vpu_debug_leave();
 640
 641         return ret;
 642 }
 643
 644 static int vidioc_querybuf(struct file *file, void *priv,
 645                            struct v4l2_buffer *buf)
 646 {
 647         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 648         int ret;
 649
 650         vpu_debug_enter();
 651
 652         switch (buf->type) {
 653         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 654                 ret = vb2_querybuf(&ctx->vq_dst, buf);
 655                 if (ret != 0) {
 656                         vpu_err("error in vb2_querybuf() for E(D)\n");
 657                         goto out;
 658                 }
 659
 660                 buf->m.planes[0].m.mem_offset += DST_QUEUE_OFF_BASE;
 661                 break;
 662
 663         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 664                 ret = vb2_querybuf(&ctx->vq_src, buf);
 665                 if (ret != 0) {
 666                         vpu_err("error in vb2_querybuf() for E(S)\n");
 667                         goto out;
 668                 }
 669                 break;
 670
 671         default:
 672                 vpu_err("invalid buf type\n");
 673                 ret = -EINVAL;
 674                 goto out;
 675         }
 676
 677 out:
 678         vpu_debug_leave();
 679
 680         return ret;
 681 }
 682
 683 static int vidioc_qbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
 684 {
 685         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 686         int ret;
 687         int i;
 688
 689         vpu_debug_enter();
 690
 691         for (i = 0; i < buf->length; i++) {
 692                 vpu_debug(4, "plane[%d]->length %d bytesused %d\n",
 693                           i, buf->m.planes[i].length,
 694                           buf->m.planes[i].bytesused);
 695         }
 696
 697         switch (buf->type) {
 698         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 699                 ret = vb2_qbuf(&ctx->vq_src, buf);
 700                 vpu_debug(4, "vb2_qbuf return %d\n", ret);
 701                 break;
 702
 703         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 704                 ret = vb2_qbuf(&ctx->vq_dst, buf);
 705                 vpu_debug(4, "vb2_qbuf return %d\n", ret);
 706                 break;
 707
 708         default:
 709                 ret = -EINVAL;
 710         }
 711
 712         vpu_debug_leave();
 713
 714         return ret;
 715 }
 716
 717 static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
 718 {
 719         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 720         int ret;
 721
 722         vpu_debug_enter();
 723
 724         switch (buf->type) {
 725         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 726                 ret = vb2_dqbuf(&ctx->vq_src, buf, file->f_flags & O_NONBLOCK);
 727                 break;
 728
 729         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 730                 ret = vb2_dqbuf(&ctx->vq_dst, buf, file->f_flags & O_NONBLOCK);
 731                 break;
 732
 733         default:
 734                 ret = -EINVAL;
 735         }
 736
 737         vpu_debug_leave();
 738
 739         return ret;
 740 }
 741
 742 static int vidioc_expbuf(struct file *file, void *priv,
 743                          struct v4l2_exportbuffer *eb)
 744 {
 745         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 746         int ret;
 747
 748         vpu_debug_enter();
 749
 750         switch (eb->type) {
 751         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 752                 ret = vb2_expbuf(&ctx->vq_src, eb);
 753                 break;
 754
 755         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 756                 ret = vb2_expbuf(&ctx->vq_dst, eb);
 757                 break;
 758
 759         default:
 760                 ret = -EINVAL;
 761         }
 762
 763         vpu_debug_leave();
 764
 765         return ret;
 766 }
 767
 768 static int vidioc_streamon(struct file *file, void *priv,
 769                            enum v4l2_buf_type type)
 770 {
 771         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 772         int ret;
 773
 774         vpu_debug_enter();
 775
 776         switch (type) {
 777         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 778                 ret = vb2_streamon(&ctx->vq_src, type);
 779                 break;
 780
 781         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 782                 ret = vb2_streamon(&ctx->vq_dst, type);
 783                 break;
 784
 785         default:
 786                 ret = -EINVAL;
 787         }
 788
 789         vpu_debug_leave();
 790
 791         return ret;
 792 }
 793
 794 static int vidioc_streamoff(struct file *file, void *priv,
 795                             enum v4l2_buf_type type)
 796 {
 797         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 798         int ret;
 799
 800         vpu_debug_enter();
 801
 802         switch (type) {
 803         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 804                 ret = vb2_streamoff(&ctx->vq_src, type);
 805                 break;
 806
 807         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 808                 ret = vb2_streamoff(&ctx->vq_dst, type);
 809                 break;
 810
 811         default:
 812                 ret = -EINVAL;
 813         }
 814
 815         vpu_debug_leave();
 816
 817         return ret;
 818 }
 819
 820 static int rk3288_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
 821 {
 822         struct rk3288_vpu_ctx *ctx = ctrl_to_ctx(ctrl);
 823         struct rk3288_vpu_dev *dev = ctx->dev;
 824         int ret = 0;
 825
 826         vpu_debug_enter();
 827
 828         vpu_debug(4, "ctrl id %d\n", ctrl->id);
 829
 830         switch (ctrl->id) {
 831         case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
 832         case V4L2_CID_MPEG_VIDEO_BITRATE:
 833         case V4L2_CID_MPEG_VIDEO_H264_MAX_QP:
 834         case V4L2_CID_MPEG_VIDEO_H264_MIN_QP:
 835         case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE:
 836         case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE:
 837         case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
 838         case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
 839         case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM:
 840         case V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE:
 841         case V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING:
 842         case V4L2_CID_MPEG_MFC51_VIDEO_RC_REACTION_COEFF:
 843         case V4L2_CID_MPEG_VIDEO_HEADER_MODE:
 844         case V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT:
 845         case V4L2_CID_MPEG_VIDEO_B_FRAMES:
 846         case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP:
 847         case V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE:
 848                 /* Ignore these controls for now. (FIXME?) */
 849                 break;
 850
 851         case V4L2_CID_PRIVATE_RK3288_HEADER:
 852         case V4L2_CID_PRIVATE_RK3288_REG_PARAMS:
 853         case V4L2_CID_PRIVATE_RK3288_HW_PARAMS:
 854                 /* Nothing to do here. The control is used directly. */
 855                 break;
 856
 857         default:
 858                 v4l2_err(&dev->v4l2_dev, "Invalid control, id=%d, val=%d\n",
 859                          ctrl->id, ctrl->val);
 860                 ret = -EINVAL;
 861         }
 862
 863         vpu_debug_leave();
 864
 865         return ret;
 866 }
 867
 868 static int rk3288_vpu_enc_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
 869 {
 870         struct rk3288_vpu_ctx *ctx = ctrl_to_ctx(ctrl);
 871         struct rk3288_vpu_dev *dev = ctx->dev;
 872         int ret = 0;
 873
 874         vpu_debug_enter();
 875
 876         vpu_debug(4, "ctrl id %d\n", ctrl->id);
 877
 878         switch (ctrl->id) {
 879         case V4L2_CID_PRIVATE_RK3288_RET_PARAMS:
 880                 memcpy(ctrl->p_new.p, ctx->run.priv_dst.cpu,
 881                         RK3288_RET_PARAMS_SIZE);
 882                 break;
 883
 884         default:
 885                 v4l2_err(&dev->v4l2_dev, "Invalid control, id=%d, val=%d\n",
 886                          ctrl->id, ctrl->val);
 887                 ret = -EINVAL;
 888         }
 889
 890         vpu_debug_leave();
 891
 892         return ret;
 893 }
 894
 895 static const struct v4l2_ctrl_ops rk3288_vpu_enc_ctrl_ops = {
 896         .s_ctrl = rk3288_vpu_enc_s_ctrl,
 897         .g_volatile_ctrl = rk3288_vpu_enc_g_volatile_ctrl,
 898 };
 899
 900 static int vidioc_cropcap(struct file *file, void *priv,
 901                           struct v4l2_cropcap *cap)
 902 {
 903         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 904         struct v4l2_pix_format_mplane *fmt = &ctx->src_fmt;
 905         int ret = 0;
 906
 907         vpu_debug_enter();
 908
 909         /* Crop only supported on source. */
 910         if (cap->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 911                 ret = -EINVAL;
 912                 goto out;
 913         }
 914
 915         cap->bounds.left = 0;
 916         cap->bounds.top = 0;
 917         cap->bounds.width = fmt->width;
 918         cap->bounds.height = fmt->height;
 919         cap->defrect = cap->bounds;
 920         cap->pixelaspect.numerator = 1;
 921         cap->pixelaspect.denominator = 1;
 922
 923 out:
 924         vpu_debug_leave();
 925
 926         return ret;
 927 }
 928
 929 static int vidioc_g_crop(struct file *file, void *priv, struct v4l2_crop *crop)
 930 {
 931         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 932         int ret = 0;
 933
 934         vpu_debug_enter();
 935
 936         /* Crop only supported on source. */
 937         if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 938                 ret = -EINVAL;
 939                 goto out;
 940         }
 941
 942         crop->c = ctx->src_crop;
 943
 944 out:
 945         vpu_debug_leave();
 946
 947         return ret;
 948 }
 949
 950 static int vidioc_s_crop(struct file *file, void *priv,
 951                          const struct v4l2_crop *crop)
 952 {
 953         struct rk3288_vpu_ctx *ctx = fh_to_ctx(priv);
 954         struct v4l2_pix_format_mplane *fmt = &ctx->src_fmt;
 955         const struct v4l2_rect *rect = &crop->c;
 956         int ret = 0;
 957
 958         vpu_debug_enter();
 959
 960         /* Crop only supported on source. */
 961         if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 962                 ret = -EINVAL;
 963                 goto out;
 964         }
 965
 966         /* Change not allowed if the queue is streaming. */
 967         if (vb2_is_streaming(&ctx->vq_src)) {
 968                 ret = -EBUSY;
 969                 goto out;
 970         }
 971
 972         /* We do not support offsets. */
 973         if (rect->left != 0 || rect->top != 0)
 974                 goto fallback;
 975
 976         /* We can crop only inside right- or bottom-most macroblocks. */
 977         if (round_up(rect->width, MB_DIM) != fmt->width
 978             || round_up(rect->height, MB_DIM) != fmt->height)
 979                 goto fallback;
 980
 981         /* We support widths aligned to 4 pixels and arbitrary heights. */
 982         ctx->src_crop.width = round_up(rect->width, 4);
 983         ctx->src_crop.height = rect->height;
 984
 985         vpu_debug_leave();
 986
 987         return 0;
 988
 989 fallback:
 990         /* Default to full frame for incorrect settings. */
 991         ctx->src_crop.width = fmt->width;
 992         ctx->src_crop.height = fmt->height;
 993
 994 out:
 995         vpu_debug_leave();
 996
 997         return ret;
 998 }
 999
1000 static const struct v4l2_ioctl_ops rk3288_vpu_enc_ioctl_ops = {
1001         .vidioc_querycap = vidioc_querycap,
1002         .vidioc_enum_fmt_vid_cap_mplane = vidioc_enum_fmt_vid_cap_mplane,
1003         .vidioc_enum_fmt_vid_out_mplane = vidioc_enum_fmt_vid_out_mplane,
1004         .vidioc_g_fmt_vid_cap_mplane = vidioc_g_fmt,
1005         .vidioc_g_fmt_vid_out_mplane = vidioc_g_fmt,
1006         .vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt,
1007         .vidioc_try_fmt_vid_out_mplane = vidioc_try_fmt,
1008         .vidioc_s_fmt_vid_cap_mplane = vidioc_s_fmt,
1009         .vidioc_s_fmt_vid_out_mplane = vidioc_s_fmt,
1010         .vidioc_reqbufs = vidioc_reqbufs,
1011         .vidioc_querybuf = vidioc_querybuf,
1012         .vidioc_qbuf = vidioc_qbuf,
1013         .vidioc_dqbuf = vidioc_dqbuf,
1014         .vidioc_expbuf = vidioc_expbuf,
1015         .vidioc_streamon = vidioc_streamon,
1016         .vidioc_streamoff = vidioc_streamoff,
1017         .vidioc_cropcap = vidioc_cropcap,
1018         .vidioc_g_crop = vidioc_g_crop,
1019         .vidioc_s_crop = vidioc_s_crop,
1020 };
1021
1022 static int rk3288_vpu_queue_setup(struct vb2_queue *vq,
1023                                   const struct v4l2_format *fmt,
1024                                   unsigned int *buf_count,
1025                                   unsigned int *plane_count,
1026                                   unsigned int psize[], void *allocators[])
1027 {
1028         struct rk3288_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1029         int ret = 0;
1030         int i;
1031
1032         vpu_debug_enter();
1033
1034         switch (vq->type) {
1035         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1036                 *plane_count = ctx->vpu_dst_fmt->num_planes;
1037
1038                 if (*buf_count < 1)
1039                         *buf_count = 1;
1040
1041                 if (*buf_count > VIDEO_MAX_FRAME)
1042                         *buf_count = VIDEO_MAX_FRAME;
1043
1044                 psize[0] = ctx->dst_fmt.plane_fmt[0].sizeimage;
1045                 /* Kernel mapping necessary for bitstream post processing. */
1046                 allocators[0] = ctx->dev->alloc_ctx_vm;
1047                 vpu_debug(0, "capture psize[%d]: %d\n", 0, psize[0]);
1048                 break;
1049
1050         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1051                 *plane_count = ctx->vpu_src_fmt->num_planes;
1052
1053                 if (*buf_count < 1)
1054                         *buf_count = 1;
1055
1056                 if (*buf_count > VIDEO_MAX_FRAME)
1057                         *buf_count = VIDEO_MAX_FRAME;
1058
1059                 for (i = 0; i < ctx->vpu_src_fmt->num_planes; ++i) {
1060                         psize[i] = ctx->src_fmt.plane_fmt[i].sizeimage;
1061                         vpu_debug(0, "output psize[%d]: %d\n", i, psize[i]);
1062                         allocators[i] = ctx->dev->alloc_ctx;
1063                 }
1064                 break;
1065
1066         default:
1067                 vpu_err("invalid queue type: %d\n", vq->type);
1068                 ret = -EINVAL;
1069         }
1070
1071         vpu_debug_leave();
1072
1073         return ret;
1074 }
1075
1076 static int rk3288_vpu_buf_prepare(struct vb2_buffer *vb)
1077 {
1078         struct vb2_queue *vq = vb->vb2_queue;
1079         struct rk3288_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1080         int ret = 0;
1081         int i;
1082
1083         vpu_debug_enter();
1084
1085         switch (vq->type) {
1086         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1087                 vpu_debug(4, "plane size: %ld, dst size: %d\n",
1088                                 vb2_plane_size(vb, 0),
1089                                 ctx->dst_fmt.plane_fmt[0].sizeimage);
1090
1091                 if (vb2_plane_size(vb, 0)
1092                     < ctx->dst_fmt.plane_fmt[0].sizeimage) {
1093                         vpu_err("plane size is too small for capture\n");
1094                         ret = -EINVAL;
1095                 }
1096                 break;
1097
1098         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1099                 for (i = 0; i < ctx->vpu_src_fmt->num_planes; ++i) {
1100                         vpu_debug(4, "plane %d size: %ld, sizeimage: %u\n", i,
1101                                         vb2_plane_size(vb, i),
1102                                         ctx->src_fmt.plane_fmt[i].sizeimage);
1103
1104                         if (vb2_plane_size(vb, i)
1105                             < ctx->src_fmt.plane_fmt[i].sizeimage) {
1106                                 vpu_err("size of plane %d is too small for output\n",
1107                                         i);
1108                                 break;
1109                         }
1110                 }
1111
1112                 if (i != ctx->vpu_src_fmt->num_planes)
1113                         ret = -EINVAL;
1114                 break;
1115
1116         default:
1117                 vpu_err("invalid queue type: %d\n", vq->type);
1118                 ret = -EINVAL;
1119         }
1120
1121         vpu_debug_leave();
1122
1123         return ret;
1124 }
1125
1126 static void rk3288_vpu_buf_finish(struct vb2_buffer *vb)
1127 {
1128         struct vb2_queue *vq = vb->vb2_queue;
1129         struct rk3288_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1130
1131         vpu_debug_enter();
1132
1133         if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE
1134             && vb->state == VB2_BUF_STATE_DONE
1135             && ctx->vpu_dst_fmt->fourcc == V4L2_PIX_FMT_VP8) {
1136                 struct rk3288_vpu_buf *buf;
1137
1138                 buf = vb_to_buf(vb);
1139                 rk3288_vpu_vp8e_assemble_bitstream(ctx, buf);
1140         }
1141
1142         vpu_debug_leave();
1143 }
1144
1145 static int rk3288_vpu_start_streaming(struct vb2_queue *q, unsigned int count)
1146 {
1147         int ret = 0;
1148         struct rk3288_vpu_ctx *ctx = fh_to_ctx(q->drv_priv);
1149         struct rk3288_vpu_dev *dev = ctx->dev;
1150         bool ready = false;
1151
1152         vpu_debug_enter();
1153
1154         if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
1155                 ret = rk3288_vpu_init(ctx);
1156                 if (ret < 0) {
1157                         vpu_err("rk3288_vpu_init failed\n");
1158                         return ret;
1159                 }
1160
1161                 ready = vb2_is_streaming(&ctx->vq_src);
1162         } else if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
1163                 ready = vb2_is_streaming(&ctx->vq_dst);
1164         }
1165
1166         if (ready)
1167                 rk3288_vpu_try_context(dev, ctx);
1168
1169         vpu_debug_leave();
1170
1171         return 0;
1172 }
1173
1174 static void rk3288_vpu_stop_streaming(struct vb2_queue *q)
1175 {
1176         unsigned long flags;
1177         struct rk3288_vpu_ctx *ctx = fh_to_ctx(q->drv_priv);
1178         struct rk3288_vpu_dev *dev = ctx->dev;
1179         struct rk3288_vpu_buf *b;
1180         LIST_HEAD(queue);
1181         int i;
1182
1183         vpu_debug_enter();
1184
1185         spin_lock_irqsave(&dev->irqlock, flags);
1186
1187         list_del_init(&ctx->list);
1188
1189         switch (q->type) {
1190         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1191                 list_splice_init(&ctx->dst_queue, &queue);
1192                 break;
1193
1194         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1195                 list_splice_init(&ctx->src_queue, &queue);
1196                 break;
1197
1198         default:
1199                 break;
1200         }
1201
1202         spin_unlock_irqrestore(&dev->irqlock, flags);
1203
1204         wait_event(dev->run_wq, dev->current_ctx != ctx);
1205
1206         while (!list_empty(&queue)) {
1207                 b = list_first_entry(&queue, struct rk3288_vpu_buf, list);
1208                 for (i = 0; i < b->b.num_planes; i++)
1209                         vb2_set_plane_payload(&b->b, i, 0);
1210                 vb2_buffer_done(&b->b, VB2_BUF_STATE_ERROR);
1211                 list_del(&b->list);
1212         }
1213
1214         if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
1215                 rk3288_vpu_deinit(ctx);
1216
1217         vpu_debug_leave();
1218 }
1219
1220 static void rk3288_vpu_buf_queue(struct vb2_buffer *vb)
1221 {
1222         struct vb2_queue *vq = vb->vb2_queue;
1223         struct rk3288_vpu_ctx *ctx = fh_to_ctx(vq->drv_priv);
1224         struct rk3288_vpu_dev *dev = ctx->dev;
1225         struct rk3288_vpu_buf *vpu_buf;
1226         unsigned long flags;
1227
1228         vpu_debug_enter();
1229
1230         switch (vq->type) {
1231         case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
1232                 vpu_buf = vb_to_buf(vb);
1233
1234                 /* Mark destination as available for use by VPU */
1235                 spin_lock_irqsave(&dev->irqlock, flags);
1236
1237                 list_add_tail(&vpu_buf->list, &ctx->dst_queue);
1238
1239                 spin_unlock_irqrestore(&dev->irqlock, flags);
1240                 break;
1241
1242         case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
1243                 vpu_buf = vb_to_buf(vb);
1244
1245                 spin_lock_irqsave(&dev->irqlock, flags);
1246
1247                 list_add_tail(&vpu_buf->list, &ctx->src_queue);
1248
1249                 spin_unlock_irqrestore(&dev->irqlock, flags);
1250                 break;
1251
1252         default:
1253                 vpu_err("unsupported buffer type (%d)\n", vq->type);
1254         }
1255
1256         if (vb2_is_streaming(&ctx->vq_src) && vb2_is_streaming(&ctx->vq_dst))
1257                 rk3288_vpu_try_context(dev, ctx);
1258
1259         vpu_debug_leave();
1260 }
1261
1262 static struct vb2_ops rk3288_vpu_enc_qops = {
1263         .queue_setup = rk3288_vpu_queue_setup,
1264         .wait_prepare = vb2_ops_wait_prepare,
1265         .wait_finish = vb2_ops_wait_finish,
1266         .buf_prepare = rk3288_vpu_buf_prepare,
1267         .buf_finish = rk3288_vpu_buf_finish,
1268         .start_streaming = rk3288_vpu_start_streaming,
1269         .stop_streaming = rk3288_vpu_stop_streaming,
1270         .buf_queue = rk3288_vpu_buf_queue,
1271 };
1272
1273 struct vb2_ops *get_enc_queue_ops(void)
1274 {
1275         return &rk3288_vpu_enc_qops;
1276 }
1277
1278 const struct v4l2_ioctl_ops *get_enc_v4l2_ioctl_ops(void)
1279 {
1280         return &rk3288_vpu_enc_ioctl_ops;
1281 }
1282
1283 static void rk3288_vpu_enc_prepare_run(struct rk3288_vpu_ctx *ctx)
1284 {
1285         struct vb2_buffer *vb2_src = &ctx->run.src->b;
1286         unsigned config_store = vb2_src->v4l2_buf.config_store;
1287
1288         v4l2_ctrl_apply_store(&ctx->ctrl_handler, config_store);
1289
1290         memcpy(ctx->run.dst->vp8e.header,
1291                 get_ctrl_ptr(ctx, RK3288_VPU_ENC_CTRL_HEADER),
1292                 RK3288_HEADER_SIZE);
1293         ctx->run.vp8e.reg_params = get_ctrl_ptr(ctx,
1294                 RK3288_VPU_ENC_CTRL_REG_PARAMS);
1295         memcpy(ctx->run.priv_src.cpu,
1296                 get_ctrl_ptr(ctx, RK3288_VPU_ENC_CTRL_HW_PARAMS),
1297                 RK3288_HW_PARAMS_SIZE);
1298 }
1299
1300 static const struct rk3288_vpu_run_ops rk3288_vpu_enc_run_ops = {
1301         .prepare_run = rk3288_vpu_enc_prepare_run,
1302 };
1303
1304 int rk3288_vpu_enc_init(struct rk3288_vpu_ctx *ctx)
1305 {
1306         struct rk3288_vpu_dev *vpu = ctx->dev;
1307         struct v4l2_format f;
1308         int ret;
1309
1310         f.fmt.pix_mp.pixelformat = DEF_SRC_FMT_ENC;
1311         ctx->vpu_src_fmt = find_format(&f, false);
1312         f.fmt.pix_mp.pixelformat = DEF_DST_FMT_ENC;
1313         ctx->vpu_dst_fmt = find_format(&f, true);
1314
1315         ret = rk3288_vpu_aux_buf_alloc(vpu, &ctx->run.priv_src,
1316                                         RK3288_HW_PARAMS_SIZE);
1317         if (ret) {
1318                 vpu_err("Failed to allocate private source buffer.\n");
1319                 return ret;
1320         }
1321
1322
1323         ret = rk3288_vpu_aux_buf_alloc(vpu, &ctx->run.priv_dst,
1324                                         RK3288_RET_PARAMS_SIZE);
1325         if (ret) {
1326                 vpu_err("Failed to allocate private destination buffer.\n");
1327                 goto err_priv_src;
1328         }
1329
1330         ret = rk3288_vpu_ctrls_setup(ctx, &rk3288_vpu_enc_ctrl_ops,
1331                                         controls, ARRAY_SIZE(controls),
1332                                         rk3288_vpu_enc_get_menu);
1333         if (ret) {
1334                 vpu_err("Failed to set up controls\n");
1335                 goto err_priv_dst;
1336         }
1337
1338         ctx->run_ops = &rk3288_vpu_enc_run_ops;
1339
1340         return 0;
1341
1342 err_priv_dst:
1343         rk3288_vpu_aux_buf_free(vpu, &ctx->run.priv_dst);
1344 err_priv_src:
1345         rk3288_vpu_aux_buf_free(vpu, &ctx->run.priv_src);
1346
1347         return ret;
1348 }
1349
1350 void rk3288_vpu_enc_exit(struct rk3288_vpu_ctx *ctx)
1351 {
1352         struct rk3288_vpu_dev *vpu = ctx->dev;
1353
1354         rk3288_vpu_ctrls_delete(ctx);
1355
1356         rk3288_vpu_aux_buf_free(vpu, &ctx->run.priv_dst);
1357         rk3288_vpu_aux_buf_free(vpu, &ctx->run.priv_src);
1358 }