2 * Rockchip RK3288 VPU codec driver
4 * Copyright (C) 2014 Google, Inc.
5 * Tomasz Figa <tfiga@chromium.org>
7 * Based on s5p-mfc driver by Samsung Electronics Co., Ltd.
9 * Copyright (C) 2011 Samsung Electronics Co., Ltd.
11 * This software is licensed under the terms of the GNU General Public
12 * License version 2, as published by the Free Software Foundation, and
13 * may be copied, distributed, and modified under those terms.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
21 #include "rk3288_vpu_common.h"
23 #include <linux/module.h>
24 #include <linux/platform_device.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/videodev2.h>
28 #include <media/v4l2-event.h>
29 #include <linux/workqueue.h>
31 #include <media/videobuf2-core.h>
32 #include <media/videobuf2-dma-contig.h>
34 #include "rk3288_vpu_dec.h"
35 #include "rk3288_vpu_enc.h"
36 #include "rk3288_vpu_hw.h"
39 module_param(debug, int, S_IRUGO | S_IWUSR);
40 MODULE_PARM_DESC(debug,
41 "Debug level - higher value produces more verbose messages");
44 * DMA coherent helpers.
47 int rk3288_vpu_aux_buf_alloc(struct rk3288_vpu_dev *vpu,
48 struct rk3288_vpu_aux_buf *buf, size_t size)
50 buf->cpu = dma_alloc_coherent(vpu->dev, size, &buf->dma, GFP_KERNEL);
58 void rk3288_vpu_aux_buf_free(struct rk3288_vpu_dev *vpu,
59 struct rk3288_vpu_aux_buf *buf)
61 dma_free_coherent(vpu->dev, buf->size, buf->cpu, buf->dma);
72 static void rk3288_vpu_prepare_run(struct rk3288_vpu_ctx *ctx)
74 if (ctx->run_ops->prepare_run)
75 ctx->run_ops->prepare_run(ctx);
78 static void __rk3288_vpu_dequeue_run_locked(struct rk3288_vpu_ctx *ctx)
80 struct rk3288_vpu_buf *src, *dst;
83 * Since ctx was dequeued from ready_ctxs list, we know that it has
84 * at least one buffer in each queue.
86 src = list_first_entry(&ctx->src_queue, struct rk3288_vpu_buf, list);
87 dst = list_first_entry(&ctx->dst_queue, struct rk3288_vpu_buf, list);
96 static void rk3288_vpu_try_run(struct rk3288_vpu_dev *dev)
98 struct rk3288_vpu_ctx *ctx = NULL;
103 spin_lock_irqsave(&dev->irqlock, flags);
105 if (list_empty(&dev->ready_ctxs) ||
106 test_bit(VPU_SUSPENDED, &dev->state))
110 if (test_and_set_bit(VPU_RUNNING, &dev->state))
112 * The hardware is already running. We will pick another
113 * run after we get the notification in rk3288_vpu_run_done().
117 ctx = list_entry(dev->ready_ctxs.next, struct rk3288_vpu_ctx, list);
118 list_del_init(&ctx->list);
120 dev->current_ctx = ctx;
121 __rk3288_vpu_dequeue_run_locked(ctx);
124 spin_unlock_irqrestore(&dev->irqlock, flags);
127 rk3288_vpu_prepare_run(ctx);
134 static void __rk3288_vpu_try_context_locked(struct rk3288_vpu_dev *dev,
135 struct rk3288_vpu_ctx *ctx)
137 if (!list_empty(&ctx->list))
138 /* Context already queued. */
141 if (!list_empty(&ctx->dst_queue) && !list_empty(&ctx->src_queue))
142 list_add_tail(&ctx->list, &dev->ready_ctxs);
145 void rk3288_vpu_run_done(struct rk3288_vpu_ctx *ctx,
146 enum vb2_buffer_state result)
148 struct vb2_buffer *src = &ctx->run.src->b;
149 struct vb2_buffer *dst = &ctx->run.dst->b;
150 struct rk3288_vpu_dev *dev = ctx->dev;
155 if (ctx->run_ops->run_done)
156 ctx->run_ops->run_done(ctx, result);
158 dst->v4l2_buf.timestamp = src->v4l2_buf.timestamp;
159 vb2_buffer_done(&ctx->run.src->b, result);
160 vb2_buffer_done(&ctx->run.dst->b, result);
162 dev->current_ctx = NULL;
163 wake_up_all(&dev->run_wq);
165 spin_lock_irqsave(&dev->irqlock, flags);
167 __rk3288_vpu_try_context_locked(dev, ctx);
168 clear_bit(VPU_RUNNING, &dev->state);
170 spin_unlock_irqrestore(&dev->irqlock, flags);
172 /* Try scheduling another run to see if we have anything left to do. */
173 rk3288_vpu_try_run(dev);
178 void rk3288_vpu_try_context(struct rk3288_vpu_dev *dev,
179 struct rk3288_vpu_ctx *ctx)
185 spin_lock_irqsave(&dev->irqlock, flags);
187 __rk3288_vpu_try_context_locked(dev, ctx);
189 spin_unlock_irqrestore(&dev->irqlock, flags);
191 rk3288_vpu_try_run(dev);
197 * Control registration.
200 #define IS_VPU_PRIV(x) ((V4L2_CTRL_ID2CLASS(x) == V4L2_CTRL_CLASS_MPEG) && \
201 V4L2_CTRL_DRIVER_PRIV(x))
203 int rk3288_vpu_ctrls_setup(struct rk3288_vpu_ctx *ctx,
204 const struct v4l2_ctrl_ops *ctrl_ops,
205 struct rk3288_vpu_control *controls,
207 const char *const *(*get_menu)(u32))
209 struct v4l2_ctrl_config cfg;
212 if (num_ctrls > ARRAY_SIZE(ctx->ctrls)) {
213 vpu_err("context control array not large enough\n");
217 v4l2_ctrl_handler_init(&ctx->ctrl_handler, num_ctrls);
218 if (ctx->ctrl_handler.error) {
219 vpu_err("v4l2_ctrl_handler_init failed\n");
220 return ctx->ctrl_handler.error;
223 for (i = 0; i < num_ctrls; i++) {
224 if (IS_VPU_PRIV(controls[i].id)
225 || controls[i].id >= V4L2_CID_CUSTOM_BASE
226 || controls[i].type == V4L2_CTRL_TYPE_PRIVATE) {
227 memset(&cfg, 0, sizeof(struct v4l2_ctrl_config));
230 cfg.id = controls[i].id;
231 cfg.min = controls[i].minimum;
232 cfg.max = controls[i].maximum;
233 cfg.max_stores = controls[i].max_stores;
234 cfg.def = controls[i].default_value;
235 cfg.name = controls[i].name;
236 cfg.type = controls[i].type;
237 cfg.elem_size = controls[i].elem_size;
238 memcpy(cfg.dims, controls[i].dims, sizeof(cfg.dims));
240 if (cfg.type == V4L2_CTRL_TYPE_MENU) {
241 cfg.menu_skip_mask = cfg.menu_skip_mask;
242 cfg.qmenu = get_menu(cfg.id);
244 cfg.step = controls[i].step;
247 ctx->ctrls[i] = v4l2_ctrl_new_custom(
248 &ctx->ctrl_handler, &cfg, NULL);
250 if (controls[i].type == V4L2_CTRL_TYPE_MENU) {
252 v4l2_ctrl_new_std_menu
262 v4l2_ctrl_new_std(&ctx->ctrl_handler,
273 if (ctx->ctrl_handler.error) {
274 vpu_err("Adding control (%d) failed\n", i);
275 return ctx->ctrl_handler.error;
278 if (controls[i].is_volatile && ctx->ctrls[i])
279 ctx->ctrls[i]->flags |= V4L2_CTRL_FLAG_VOLATILE;
280 if (controls[i].is_read_only && ctx->ctrls[i])
281 ctx->ctrls[i]->flags |= V4L2_CTRL_FLAG_READ_ONLY;
282 if (controls[i].can_store && ctx->ctrls[i])
283 ctx->ctrls[i]->flags |= V4L2_CTRL_FLAG_CAN_STORE;
286 v4l2_ctrl_handler_setup(&ctx->ctrl_handler);
287 ctx->num_ctrls = num_ctrls;
291 void rk3288_vpu_ctrls_delete(struct rk3288_vpu_ctx *ctx)
295 v4l2_ctrl_handler_free(&ctx->ctrl_handler);
296 for (i = 0; i < ctx->num_ctrls; i++)
297 ctx->ctrls[i] = NULL;
301 * V4L2 file operations.
304 static int rk3288_vpu_open(struct file *filp)
306 struct video_device *vdev = video_devdata(filp);
307 struct rk3288_vpu_dev *dev = video_drvdata(filp);
308 struct rk3288_vpu_ctx *ctx = NULL;
313 * We do not need any extra locking here, because we operate only
314 * on local data here, except reading few fields from dev, which
315 * do not change through device's lifetime (which is guaranteed by
316 * reference on module from open()) and V4L2 internal objects (such
317 * as vdev and ctx->fh), which have proper locking done in respective
318 * helper functions used here.
323 /* Allocate memory for context */
324 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
330 v4l2_fh_init(&ctx->fh, video_devdata(filp));
331 filp->private_data = &ctx->fh;
332 v4l2_fh_add(&ctx->fh);
334 INIT_LIST_HEAD(&ctx->src_queue);
335 INIT_LIST_HEAD(&ctx->dst_queue);
336 INIT_LIST_HEAD(&ctx->list);
338 if (vdev == dev->vfd_enc) {
339 /* only for encoder */
340 ret = rk3288_vpu_enc_init(ctx);
342 vpu_err("Failed to initialize encoder context\n");
345 } else if (vdev == dev->vfd_dec) {
346 /* only for decoder */
347 ret = rk3288_vpu_dec_init(ctx);
349 vpu_err("Failed to initialize decoder context\n");
356 ctx->fh.ctrl_handler = &ctx->ctrl_handler;
358 /* Init videobuf2 queue for CAPTURE */
360 q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
361 q->drv_priv = &ctx->fh;
362 q->io_modes = VB2_MMAP | VB2_USERPTR;
363 q->lock = &dev->vpu_mutex;
364 q->buf_struct_size = sizeof(struct rk3288_vpu_buf);
366 if (vdev == dev->vfd_enc)
367 q->ops = get_enc_queue_ops();
368 else if (vdev == dev->vfd_dec)
369 q->ops = get_dec_queue_ops();
371 q->mem_ops = &vb2_dma_contig_memops;
372 q->timestamp_type = V4L2_BUF_FLAG_TIMESTAMP_COPY;
374 ret = vb2_queue_init(q);
376 vpu_err("Failed to initialize videobuf2 queue(capture)\n");
377 goto err_enc_dec_exit;
380 /* Init videobuf2 queue for OUTPUT */
382 q->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
383 q->drv_priv = &ctx->fh;
384 q->io_modes = VB2_MMAP | VB2_USERPTR;
385 q->lock = &dev->vpu_mutex;
386 q->buf_struct_size = sizeof(struct rk3288_vpu_buf);
388 if (vdev == dev->vfd_enc)
389 q->ops = get_enc_queue_ops();
390 else if (vdev == dev->vfd_dec)
391 q->ops = get_dec_queue_ops();
393 q->mem_ops = &vb2_dma_contig_memops;
394 q->timestamp_type = V4L2_BUF_FLAG_TIMESTAMP_COPY;
396 ret = vb2_queue_init(q);
398 vpu_err("Failed to initialize videobuf2 queue(output)\n");
399 goto err_vq_dst_release;
407 vb2_queue_release(&ctx->vq_dst);
409 if (vdev == dev->vfd_enc)
410 rk3288_vpu_enc_exit(ctx);
411 else if (vdev == dev->vfd_dec)
412 rk3288_vpu_dec_exit(ctx);
414 v4l2_fh_del(&ctx->fh);
415 v4l2_fh_exit(&ctx->fh);
423 static int rk3288_vpu_release(struct file *filp)
425 struct rk3288_vpu_ctx *ctx = fh_to_ctx(filp->private_data);
426 struct video_device *vdev = video_devdata(filp);
427 struct rk3288_vpu_dev *dev = ctx->dev;
430 * No need for extra locking because this was the last reference
437 * vb2_queue_release() ensures that streaming is stopped, which
438 * in turn means that there are no frames still being processed
441 vb2_queue_release(&ctx->vq_src);
442 vb2_queue_release(&ctx->vq_dst);
444 v4l2_fh_del(&ctx->fh);
445 v4l2_fh_exit(&ctx->fh);
447 if (vdev == dev->vfd_enc)
448 rk3288_vpu_enc_exit(ctx);
449 else if (vdev == dev->vfd_dec)
450 rk3288_vpu_dec_exit(ctx);
459 static unsigned int rk3288_vpu_poll(struct file *filp,
460 struct poll_table_struct *wait)
462 struct rk3288_vpu_ctx *ctx = fh_to_ctx(filp->private_data);
463 struct vb2_queue *src_q, *dst_q;
464 struct vb2_buffer *src_vb = NULL, *dst_vb = NULL;
470 src_q = &ctx->vq_src;
471 dst_q = &ctx->vq_dst;
474 * There has to be at least one buffer queued on each queued_list, which
475 * means either in driver already or waiting for driver to claim it
476 * and start processing.
478 if ((!vb2_is_streaming(src_q) || list_empty(&src_q->queued_list)) &&
479 (!vb2_is_streaming(dst_q) || list_empty(&dst_q->queued_list))) {
480 vpu_debug(0, "src q streaming %d, dst q streaming %d, src list empty(%d), dst list empty(%d)\n",
481 src_q->streaming, dst_q->streaming,
482 list_empty(&src_q->queued_list),
483 list_empty(&dst_q->queued_list));
487 poll_wait(filp, &ctx->fh.wait, wait);
488 poll_wait(filp, &src_q->done_wq, wait);
489 poll_wait(filp, &dst_q->done_wq, wait);
491 if (v4l2_event_pending(&ctx->fh))
494 spin_lock_irqsave(&src_q->done_lock, flags);
496 if (!list_empty(&src_q->done_list))
497 src_vb = list_first_entry(&src_q->done_list, struct vb2_buffer,
500 if (src_vb && (src_vb->state == VB2_BUF_STATE_DONE ||
501 src_vb->state == VB2_BUF_STATE_ERROR))
502 rc |= POLLOUT | POLLWRNORM;
504 spin_unlock_irqrestore(&src_q->done_lock, flags);
506 spin_lock_irqsave(&dst_q->done_lock, flags);
508 if (!list_empty(&dst_q->done_list))
509 dst_vb = list_first_entry(&dst_q->done_list, struct vb2_buffer,
512 if (dst_vb && (dst_vb->state == VB2_BUF_STATE_DONE ||
513 dst_vb->state == VB2_BUF_STATE_ERROR))
514 rc |= POLLIN | POLLRDNORM;
516 spin_unlock_irqrestore(&dst_q->done_lock, flags);
521 static int rk3288_vpu_mmap(struct file *filp, struct vm_area_struct *vma)
523 struct rk3288_vpu_ctx *ctx = fh_to_ctx(filp->private_data);
524 unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
529 if (offset < DST_QUEUE_OFF_BASE) {
530 vpu_debug(4, "mmaping source\n");
532 ret = vb2_mmap(&ctx->vq_src, vma);
533 } else { /* capture */
534 vpu_debug(4, "mmaping destination\n");
536 vma->vm_pgoff -= (DST_QUEUE_OFF_BASE >> PAGE_SHIFT);
537 ret = vb2_mmap(&ctx->vq_dst, vma);
545 static const struct v4l2_file_operations rk3288_vpu_fops = {
546 .owner = THIS_MODULE,
547 .open = rk3288_vpu_open,
548 .release = rk3288_vpu_release,
549 .poll = rk3288_vpu_poll,
550 .unlocked_ioctl = video_ioctl2,
551 .mmap = rk3288_vpu_mmap,
558 static int rk3288_vpu_probe(struct platform_device *pdev)
560 struct rk3288_vpu_dev *vpu = NULL;
561 DEFINE_DMA_ATTRS(attrs_novm);
562 struct video_device *vfd;
567 vpu = devm_kzalloc(&pdev->dev, sizeof(*vpu), GFP_KERNEL);
571 vpu->dev = &pdev->dev;
573 mutex_init(&vpu->vpu_mutex);
574 spin_lock_init(&vpu->irqlock);
575 INIT_LIST_HEAD(&vpu->ready_ctxs);
576 init_waitqueue_head(&vpu->run_wq);
578 ret = rk3288_vpu_hw_probe(vpu);
580 dev_err(&pdev->dev, "vcodec_hw_probe failed\n");
584 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs_novm);
585 vpu->alloc_ctx = vb2_dma_contig_init_ctx_attrs(&pdev->dev,
587 if (IS_ERR(vpu->alloc_ctx)) {
588 ret = PTR_ERR(vpu->alloc_ctx);
592 vpu->alloc_ctx_vm = vb2_dma_contig_init_ctx(&pdev->dev);
593 if (IS_ERR(vpu->alloc_ctx_vm)) {
594 ret = PTR_ERR(vpu->alloc_ctx_vm);
595 goto err_dma_contig_vm;
598 ret = v4l2_device_register(&pdev->dev, &vpu->v4l2_dev);
600 dev_err(&pdev->dev, "Failed to register v4l2 device\n");
601 goto err_v4l2_dev_reg;
604 platform_set_drvdata(pdev, vpu);
607 vfd = video_device_alloc();
609 v4l2_err(&vpu->v4l2_dev, "Failed to allocate video device\n");
614 vfd->fops = &rk3288_vpu_fops;
615 vfd->ioctl_ops = get_enc_v4l2_ioctl_ops();
616 vfd->release = video_device_release;
617 vfd->lock = &vpu->vpu_mutex;
618 vfd->v4l2_dev = &vpu->v4l2_dev;
619 vfd->vfl_dir = VFL_DIR_M2M;
620 snprintf(vfd->name, sizeof(vfd->name), "%s", RK3288_VPU_ENC_NAME);
623 video_set_drvdata(vfd, vpu);
625 ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
627 v4l2_err(&vpu->v4l2_dev, "Failed to register video device\n");
628 video_device_release(vfd);
632 v4l2_info(&vpu->v4l2_dev,
633 "Rockchip RK3288 VPU encoder registered as /vpu/video%d\n",
637 vfd = video_device_alloc();
639 v4l2_err(&vpu->v4l2_dev, "Failed to allocate video device\n");
644 vfd->fops = &rk3288_vpu_fops;
645 vfd->ioctl_ops = get_dec_v4l2_ioctl_ops();
646 vfd->release = video_device_release;
647 vfd->lock = &vpu->vpu_mutex;
648 vfd->v4l2_dev = &vpu->v4l2_dev;
649 vfd->vfl_dir = VFL_DIR_M2M;
650 snprintf(vfd->name, sizeof(vfd->name), "%s", RK3288_VPU_DEC_NAME);
653 video_set_drvdata(vfd, vpu);
655 ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
657 v4l2_err(&vpu->v4l2_dev, "Failed to register video device\n");
658 video_device_release(vfd);
662 v4l2_info(&vpu->v4l2_dev,
663 "Rockchip RK3288 VPU decoder registered as /vpu/video%d\n",
671 video_device_release(vpu->vfd_dec);
673 video_unregister_device(vpu->vfd_enc);
675 video_device_release(vpu->vfd_enc);
677 v4l2_device_unregister(&vpu->v4l2_dev);
679 vb2_dma_contig_cleanup_ctx(vpu->alloc_ctx_vm);
681 vb2_dma_contig_cleanup_ctx(vpu->alloc_ctx);
683 rk3288_vpu_hw_remove(vpu);
685 pr_debug("%s-- with error\n", __func__);
691 static int rk3288_vpu_remove(struct platform_device *pdev)
693 struct rk3288_vpu_dev *vpu = platform_get_drvdata(pdev);
697 v4l2_info(&vpu->v4l2_dev, "Removing %s\n", pdev->name);
700 * We are safe here assuming that .remove() got called as
701 * a result of module removal, which guarantees that all
702 * contexts have been released.
705 video_unregister_device(vpu->vfd_dec);
706 video_unregister_device(vpu->vfd_enc);
707 v4l2_device_unregister(&vpu->v4l2_dev);
708 vb2_dma_contig_cleanup_ctx(vpu->alloc_ctx_vm);
709 vb2_dma_contig_cleanup_ctx(vpu->alloc_ctx);
710 rk3288_vpu_hw_remove(vpu);
717 static struct platform_device_id vpu_driver_ids[] = {
718 { .name = "rk3288-vpu", },
722 MODULE_DEVICE_TABLE(platform, vpu_driver_ids);
725 static const struct of_device_id of_rk3288_vpu_match[] = {
726 { .compatible = "rockchip,rk3288-vpu", },
729 MODULE_DEVICE_TABLE(of, of_rk3288_vpu_match);
732 #ifdef CONFIG_PM_SLEEP
733 static int rk3288_vpu_suspend(struct device *dev)
735 struct rk3288_vpu_dev *vpu = dev_get_drvdata(dev);
737 set_bit(VPU_SUSPENDED, &vpu->state);
738 wait_event(vpu->run_wq, vpu->current_ctx == NULL);
743 static int rk3288_vpu_resume(struct device *dev)
745 struct rk3288_vpu_dev *vpu = dev_get_drvdata(dev);
747 clear_bit(VPU_SUSPENDED, &vpu->state);
748 rk3288_vpu_try_run(vpu);
754 static const struct dev_pm_ops rk3288_vpu_pm_ops = {
755 SET_SYSTEM_SLEEP_PM_OPS(rk3288_vpu_suspend, rk3288_vpu_resume)
758 static struct platform_driver rk3288_vpu_driver = {
759 .probe = rk3288_vpu_probe,
760 .remove = rk3288_vpu_remove,
761 .id_table = vpu_driver_ids,
763 .name = RK3288_VPU_NAME,
764 .owner = THIS_MODULE,
765 .of_match_table = of_match_ptr(of_rk3288_vpu_match),
766 .pm = &rk3288_vpu_pm_ops,
769 module_platform_driver(rk3288_vpu_driver);
771 MODULE_LICENSE("GPL v2");
772 MODULE_AUTHOR("Alpha Lin <Alpha.Lin@Rock-Chips.com>");
773 MODULE_AUTHOR("Tomasz Figa <tfiga@chromium.org>");
774 MODULE_DESCRIPTION("Rockchip RK3288 VPU codec driver");