2 * Video Capture Subdev for Freescale i.MX5/6 SOC
4 * Copyright (c) 2012-2016 Mentor Graphics Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 #include <linux/delay.h>
13 #include <linux/module.h>
14 #include <linux/of_platform.h>
15 #include <linux/pinctrl/consumer.h>
16 #include <linux/platform_device.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/spinlock.h>
20 #include <linux/timer.h>
21 #include <media/v4l2-ctrls.h>
22 #include <media/v4l2-device.h>
23 #include <media/v4l2-event.h>
24 #include <media/v4l2-fwnode.h>
25 #include <media/v4l2-ioctl.h>
26 #include <media/v4l2-mc.h>
27 #include <media/v4l2-subdev.h>
28 #include <media/videobuf2-dma-contig.h>
29 #include <video/imx-ipu-v3.h>
30 #include <media/imx.h>
31 #include "imx-media.h"
34 struct imx_media_video_dev vdev;
36 struct v4l2_subdev *src_sd;
40 struct imx_media_dev *md;
42 struct media_pad vdev_pad;
44 struct mutex mutex; /* capture device mutex */
46 /* the videobuf2 queue */
48 /* list of ready imx_media_buffer's from q */
49 struct list_head ready_q;
53 /* controls inherited from subdevs */
54 struct v4l2_ctrl_handler ctrl_hdlr;
57 bool stop; /* streaming is stopping */
60 #define to_capture_priv(v) container_of(v, struct capture_priv, vdev)
62 /* In bytes, per queue */
63 #define VID_MEM_LIMIT SZ_64M
65 static const struct vb2_ops capture_qops;
71 static int vidioc_querycap(struct file *file, void *fh,
72 struct v4l2_capability *cap)
74 struct capture_priv *priv = video_drvdata(file);
76 strncpy(cap->driver, "imx-media-capture", sizeof(cap->driver) - 1);
77 strncpy(cap->card, "imx-media-capture", sizeof(cap->card) - 1);
78 snprintf(cap->bus_info, sizeof(cap->bus_info),
79 "platform:%s", priv->src_sd->name);
84 static int capture_enum_framesizes(struct file *file, void *fh,
85 struct v4l2_frmsizeenum *fsize)
87 struct capture_priv *priv = video_drvdata(file);
88 const struct imx_media_pixfmt *cc;
89 struct v4l2_subdev_frame_size_enum fse = {
90 .index = fsize->index,
91 .pad = priv->src_sd_pad,
92 .which = V4L2_SUBDEV_FORMAT_ACTIVE,
96 cc = imx_media_find_format(fsize->pixel_format, CS_SEL_ANY, true);
100 fse.code = cc->codes[0];
102 ret = v4l2_subdev_call(priv->src_sd, pad, enum_frame_size, NULL, &fse);
106 if (fse.min_width == fse.max_width &&
107 fse.min_height == fse.max_height) {
108 fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
109 fsize->discrete.width = fse.min_width;
110 fsize->discrete.height = fse.min_height;
112 fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
113 fsize->stepwise.min_width = fse.min_width;
114 fsize->stepwise.max_width = fse.max_width;
115 fsize->stepwise.min_height = fse.min_height;
116 fsize->stepwise.max_height = fse.max_height;
117 fsize->stepwise.step_width = 1;
118 fsize->stepwise.step_height = 1;
124 static int capture_enum_frameintervals(struct file *file, void *fh,
125 struct v4l2_frmivalenum *fival)
127 struct capture_priv *priv = video_drvdata(file);
128 const struct imx_media_pixfmt *cc;
129 struct v4l2_subdev_frame_interval_enum fie = {
130 .index = fival->index,
131 .pad = priv->src_sd_pad,
132 .width = fival->width,
133 .height = fival->height,
134 .which = V4L2_SUBDEV_FORMAT_ACTIVE,
138 cc = imx_media_find_format(fival->pixel_format, CS_SEL_ANY, true);
142 fie.code = cc->codes[0];
144 ret = v4l2_subdev_call(priv->src_sd, pad, enum_frame_interval, NULL, &fie);
148 fival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
149 fival->discrete = fie.interval;
154 static int capture_enum_fmt_vid_cap(struct file *file, void *fh,
155 struct v4l2_fmtdesc *f)
157 struct capture_priv *priv = video_drvdata(file);
158 const struct imx_media_pixfmt *cc_src;
159 struct v4l2_subdev_format fmt_src;
163 fmt_src.pad = priv->src_sd_pad;
164 fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
165 ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
167 v4l2_err(priv->src_sd, "failed to get src_sd format\n");
171 cc_src = imx_media_find_ipu_format(fmt_src.format.code, CS_SEL_ANY);
173 cc_src = imx_media_find_mbus_format(fmt_src.format.code,
181 fourcc = cc_src->fourcc;
183 u32 cs_sel = (cc_src->cs == IPUV3_COLORSPACE_YUV) ?
184 CS_SEL_YUV : CS_SEL_RGB;
186 ret = imx_media_enum_format(&fourcc, f->index, cs_sel);
191 f->pixelformat = fourcc;
196 static int capture_g_fmt_vid_cap(struct file *file, void *fh,
197 struct v4l2_format *f)
199 struct capture_priv *priv = video_drvdata(file);
206 static int capture_try_fmt_vid_cap(struct file *file, void *fh,
207 struct v4l2_format *f)
209 struct capture_priv *priv = video_drvdata(file);
210 struct v4l2_subdev_format fmt_src;
211 const struct imx_media_pixfmt *cc, *cc_src;
214 fmt_src.pad = priv->src_sd_pad;
215 fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
216 ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
220 cc_src = imx_media_find_ipu_format(fmt_src.format.code, CS_SEL_ANY);
222 cc_src = imx_media_find_mbus_format(fmt_src.format.code,
232 cs_sel = (cc_src->cs == IPUV3_COLORSPACE_YUV) ?
233 CS_SEL_YUV : CS_SEL_RGB;
234 fourcc = f->fmt.pix.pixelformat;
236 cc = imx_media_find_format(fourcc, cs_sel, false);
238 imx_media_enum_format(&fourcc, 0, cs_sel);
239 cc = imx_media_find_format(fourcc, cs_sel, false);
243 imx_media_mbus_fmt_to_pix_fmt(&f->fmt.pix, &fmt_src.format, cc);
248 static int capture_s_fmt_vid_cap(struct file *file, void *fh,
249 struct v4l2_format *f)
251 struct capture_priv *priv = video_drvdata(file);
254 if (vb2_is_busy(&priv->q)) {
255 v4l2_err(priv->src_sd, "%s queue busy\n", __func__);
259 ret = capture_try_fmt_vid_cap(file, priv, f);
263 priv->vdev.fmt.fmt.pix = f->fmt.pix;
264 priv->vdev.cc = imx_media_find_format(f->fmt.pix.pixelformat,
270 static int capture_querystd(struct file *file, void *fh, v4l2_std_id *std)
272 struct capture_priv *priv = video_drvdata(file);
274 return v4l2_subdev_call(priv->src_sd, video, querystd, std);
277 static int capture_g_std(struct file *file, void *fh, v4l2_std_id *std)
279 struct capture_priv *priv = video_drvdata(file);
281 return v4l2_subdev_call(priv->src_sd, video, g_std, std);
284 static int capture_s_std(struct file *file, void *fh, v4l2_std_id std)
286 struct capture_priv *priv = video_drvdata(file);
288 if (vb2_is_busy(&priv->q))
291 return v4l2_subdev_call(priv->src_sd, video, s_std, std);
294 static int capture_g_parm(struct file *file, void *fh,
295 struct v4l2_streamparm *a)
297 struct capture_priv *priv = video_drvdata(file);
298 struct v4l2_subdev_frame_interval fi;
301 if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
304 memset(&fi, 0, sizeof(fi));
305 fi.pad = priv->src_sd_pad;
306 ret = v4l2_subdev_call(priv->src_sd, video, g_frame_interval, &fi);
310 a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
311 a->parm.capture.timeperframe = fi.interval;
316 static int capture_s_parm(struct file *file, void *fh,
317 struct v4l2_streamparm *a)
319 struct capture_priv *priv = video_drvdata(file);
320 struct v4l2_subdev_frame_interval fi;
323 if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
326 memset(&fi, 0, sizeof(fi));
327 fi.pad = priv->src_sd_pad;
328 fi.interval = a->parm.capture.timeperframe;
329 ret = v4l2_subdev_call(priv->src_sd, video, s_frame_interval, &fi);
333 a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
334 a->parm.capture.timeperframe = fi.interval;
339 static const struct v4l2_ioctl_ops capture_ioctl_ops = {
340 .vidioc_querycap = vidioc_querycap,
342 .vidioc_enum_framesizes = capture_enum_framesizes,
343 .vidioc_enum_frameintervals = capture_enum_frameintervals,
345 .vidioc_enum_fmt_vid_cap = capture_enum_fmt_vid_cap,
346 .vidioc_g_fmt_vid_cap = capture_g_fmt_vid_cap,
347 .vidioc_try_fmt_vid_cap = capture_try_fmt_vid_cap,
348 .vidioc_s_fmt_vid_cap = capture_s_fmt_vid_cap,
350 .vidioc_querystd = capture_querystd,
351 .vidioc_g_std = capture_g_std,
352 .vidioc_s_std = capture_s_std,
354 .vidioc_g_parm = capture_g_parm,
355 .vidioc_s_parm = capture_s_parm,
357 .vidioc_reqbufs = vb2_ioctl_reqbufs,
358 .vidioc_create_bufs = vb2_ioctl_create_bufs,
359 .vidioc_prepare_buf = vb2_ioctl_prepare_buf,
360 .vidioc_querybuf = vb2_ioctl_querybuf,
361 .vidioc_qbuf = vb2_ioctl_qbuf,
362 .vidioc_dqbuf = vb2_ioctl_dqbuf,
363 .vidioc_expbuf = vb2_ioctl_expbuf,
364 .vidioc_streamon = vb2_ioctl_streamon,
365 .vidioc_streamoff = vb2_ioctl_streamoff,
372 static int capture_queue_setup(struct vb2_queue *vq,
373 unsigned int *nbuffers,
374 unsigned int *nplanes,
375 unsigned int sizes[],
376 struct device *alloc_devs[])
378 struct capture_priv *priv = vb2_get_drv_priv(vq);
379 struct v4l2_pix_format *pix = &priv->vdev.fmt.fmt.pix;
380 unsigned int count = *nbuffers;
382 if (vq->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
386 if (*nplanes != 1 || sizes[0] < pix->sizeimage)
388 count += vq->num_buffers;
391 count = min_t(__u32, VID_MEM_LIMIT / pix->sizeimage, count);
394 *nbuffers = (count < vq->num_buffers) ? 0 :
395 count - vq->num_buffers;
400 sizes[0] = pix->sizeimage;
405 static int capture_buf_init(struct vb2_buffer *vb)
407 struct imx_media_buffer *buf = to_imx_media_vb(vb);
409 INIT_LIST_HEAD(&buf->list);
414 static int capture_buf_prepare(struct vb2_buffer *vb)
416 struct vb2_queue *vq = vb->vb2_queue;
417 struct capture_priv *priv = vb2_get_drv_priv(vq);
418 struct v4l2_pix_format *pix = &priv->vdev.fmt.fmt.pix;
420 if (vb2_plane_size(vb, 0) < pix->sizeimage) {
421 v4l2_err(priv->src_sd,
422 "data will not fit into plane (%lu < %lu)\n",
423 vb2_plane_size(vb, 0), (long)pix->sizeimage);
427 vb2_set_plane_payload(vb, 0, pix->sizeimage);
432 static void capture_buf_queue(struct vb2_buffer *vb)
434 struct capture_priv *priv = vb2_get_drv_priv(vb->vb2_queue);
435 struct imx_media_buffer *buf = to_imx_media_vb(vb);
438 spin_lock_irqsave(&priv->q_lock, flags);
440 list_add_tail(&buf->list, &priv->ready_q);
442 spin_unlock_irqrestore(&priv->q_lock, flags);
445 static int capture_start_streaming(struct vb2_queue *vq, unsigned int count)
447 struct capture_priv *priv = vb2_get_drv_priv(vq);
448 struct imx_media_buffer *buf, *tmp;
452 if (vb2_is_streaming(vq))
455 ret = imx_media_pipeline_set_stream(priv->md, &priv->src_sd->entity,
458 v4l2_err(priv->src_sd, "pipeline start failed with %d\n", ret);
467 spin_lock_irqsave(&priv->q_lock, flags);
468 list_for_each_entry_safe(buf, tmp, &priv->ready_q, list) {
469 list_del(&buf->list);
470 vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_QUEUED);
472 spin_unlock_irqrestore(&priv->q_lock, flags);
476 static void capture_stop_streaming(struct vb2_queue *vq)
478 struct capture_priv *priv = vb2_get_drv_priv(vq);
479 struct imx_media_buffer *frame;
483 if (!vb2_is_streaming(vq))
486 spin_lock_irqsave(&priv->q_lock, flags);
488 spin_unlock_irqrestore(&priv->q_lock, flags);
490 ret = imx_media_pipeline_set_stream(priv->md, &priv->src_sd->entity,
493 v4l2_warn(priv->src_sd, "pipeline stop failed with %d\n", ret);
495 /* release all active buffers */
496 spin_lock_irqsave(&priv->q_lock, flags);
497 while (!list_empty(&priv->ready_q)) {
498 frame = list_entry(priv->ready_q.next,
499 struct imx_media_buffer, list);
500 list_del(&frame->list);
501 vb2_buffer_done(&frame->vbuf.vb2_buf, VB2_BUF_STATE_ERROR);
503 spin_unlock_irqrestore(&priv->q_lock, flags);
506 static const struct vb2_ops capture_qops = {
507 .queue_setup = capture_queue_setup,
508 .buf_init = capture_buf_init,
509 .buf_prepare = capture_buf_prepare,
510 .buf_queue = capture_buf_queue,
511 .wait_prepare = vb2_ops_wait_prepare,
512 .wait_finish = vb2_ops_wait_finish,
513 .start_streaming = capture_start_streaming,
514 .stop_streaming = capture_stop_streaming,
520 static int capture_open(struct file *file)
522 struct capture_priv *priv = video_drvdata(file);
523 struct video_device *vfd = priv->vdev.vfd;
526 if (mutex_lock_interruptible(&priv->mutex))
529 ret = v4l2_fh_open(file);
531 v4l2_err(priv->src_sd, "v4l2_fh_open failed\n");
533 ret = v4l2_pipeline_pm_use(&vfd->entity, 1);
535 v4l2_fh_release(file);
537 mutex_unlock(&priv->mutex);
541 static int capture_release(struct file *file)
543 struct capture_priv *priv = video_drvdata(file);
544 struct video_device *vfd = priv->vdev.vfd;
545 struct vb2_queue *vq = &priv->q;
548 mutex_lock(&priv->mutex);
550 if (file->private_data == vq->owner) {
551 vb2_queue_release(vq);
555 v4l2_pipeline_pm_use(&vfd->entity, 0);
557 v4l2_fh_release(file);
558 mutex_unlock(&priv->mutex);
562 static const struct v4l2_file_operations capture_fops = {
563 .owner = THIS_MODULE,
564 .open = capture_open,
565 .release = capture_release,
566 .poll = vb2_fop_poll,
567 .unlocked_ioctl = video_ioctl2,
568 .mmap = vb2_fop_mmap,
571 static struct video_device capture_videodev = {
572 .fops = &capture_fops,
573 .ioctl_ops = &capture_ioctl_ops,
575 .release = video_device_release,
576 .vfl_dir = VFL_DIR_RX,
577 .tvnorms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
578 .device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING,
581 void imx_media_capture_device_set_format(struct imx_media_video_dev *vdev,
582 struct v4l2_pix_format *pix)
584 struct capture_priv *priv = to_capture_priv(vdev);
586 mutex_lock(&priv->mutex);
587 priv->vdev.fmt.fmt.pix = *pix;
588 priv->vdev.cc = imx_media_find_format(pix->pixelformat, CS_SEL_ANY,
590 mutex_unlock(&priv->mutex);
592 EXPORT_SYMBOL_GPL(imx_media_capture_device_set_format);
594 struct imx_media_buffer *
595 imx_media_capture_device_next_buf(struct imx_media_video_dev *vdev)
597 struct capture_priv *priv = to_capture_priv(vdev);
598 struct imx_media_buffer *buf = NULL;
601 spin_lock_irqsave(&priv->q_lock, flags);
603 /* get next queued buffer */
604 if (!list_empty(&priv->ready_q)) {
605 buf = list_entry(priv->ready_q.next, struct imx_media_buffer,
607 list_del(&buf->list);
610 spin_unlock_irqrestore(&priv->q_lock, flags);
614 EXPORT_SYMBOL_GPL(imx_media_capture_device_next_buf);
616 void imx_media_capture_device_error(struct imx_media_video_dev *vdev)
618 struct capture_priv *priv = to_capture_priv(vdev);
619 struct vb2_queue *vq = &priv->q;
622 if (!vb2_is_streaming(vq))
625 spin_lock_irqsave(&priv->q_lock, flags);
627 spin_unlock_irqrestore(&priv->q_lock, flags);
629 EXPORT_SYMBOL_GPL(imx_media_capture_device_error);
631 int imx_media_capture_device_register(struct imx_media_video_dev *vdev)
633 struct capture_priv *priv = to_capture_priv(vdev);
634 struct v4l2_subdev *sd = priv->src_sd;
635 struct video_device *vfd = vdev->vfd;
636 struct vb2_queue *vq = &priv->q;
637 struct v4l2_subdev_format fmt_src;
640 /* get media device */
641 priv->md = dev_get_drvdata(sd->v4l2_dev->dev);
643 vfd->v4l2_dev = sd->v4l2_dev;
645 ret = video_register_device(vfd, VFL_TYPE_GRABBER, -1);
647 v4l2_err(sd, "Failed to register video device\n");
651 vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
652 vq->io_modes = VB2_MMAP | VB2_DMABUF;
654 vq->buf_struct_size = sizeof(struct imx_media_buffer);
655 vq->ops = &capture_qops;
656 vq->mem_ops = &vb2_dma_contig_memops;
657 vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
658 vq->lock = &priv->mutex;
659 vq->min_buffers_needed = 2;
662 ret = vb2_queue_init(vq);
664 v4l2_err(sd, "vb2_queue_init failed\n");
668 INIT_LIST_HEAD(&priv->ready_q);
670 priv->vdev_pad.flags = MEDIA_PAD_FL_SINK;
671 ret = media_entity_pads_init(&vfd->entity, 1, &priv->vdev_pad);
673 v4l2_err(sd, "failed to init dev pad\n");
677 /* create the link from the src_sd devnode pad to device node */
678 ret = media_create_pad_link(&sd->entity, priv->src_sd_pad,
681 v4l2_err(sd, "failed to create link to device node\n");
685 /* setup default format */
686 fmt_src.pad = priv->src_sd_pad;
687 fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
688 ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt_src);
690 v4l2_err(sd, "failed to get src_sd format\n");
694 vdev->fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
695 imx_media_mbus_fmt_to_pix_fmt(&vdev->fmt.fmt.pix,
696 &fmt_src.format, NULL);
697 vdev->cc = imx_media_find_format(vdev->fmt.fmt.pix.pixelformat,
700 v4l2_info(sd, "Registered %s as /dev/%s\n", vfd->name,
701 video_device_node_name(vfd));
703 vfd->ctrl_handler = &priv->ctrl_hdlr;
707 video_unregister_device(vfd);
710 EXPORT_SYMBOL_GPL(imx_media_capture_device_register);
712 void imx_media_capture_device_unregister(struct imx_media_video_dev *vdev)
714 struct capture_priv *priv = to_capture_priv(vdev);
715 struct video_device *vfd = priv->vdev.vfd;
717 mutex_lock(&priv->mutex);
719 if (video_is_registered(vfd)) {
720 video_unregister_device(vfd);
721 media_entity_cleanup(&vfd->entity);
724 mutex_unlock(&priv->mutex);
726 EXPORT_SYMBOL_GPL(imx_media_capture_device_unregister);
728 struct imx_media_video_dev *
729 imx_media_capture_device_init(struct v4l2_subdev *src_sd, int pad)
731 struct capture_priv *priv;
732 struct video_device *vfd;
734 priv = devm_kzalloc(src_sd->dev, sizeof(*priv), GFP_KERNEL);
736 return ERR_PTR(-ENOMEM);
738 priv->src_sd = src_sd;
739 priv->src_sd_pad = pad;
740 priv->dev = src_sd->dev;
742 mutex_init(&priv->mutex);
743 spin_lock_init(&priv->q_lock);
745 snprintf(capture_videodev.name, sizeof(capture_videodev.name),
746 "%s capture", src_sd->name);
748 vfd = video_device_alloc();
750 return ERR_PTR(-ENOMEM);
752 *vfd = capture_videodev;
753 vfd->lock = &priv->mutex;
754 vfd->queue = &priv->q;
755 priv->vdev.vfd = vfd;
757 video_set_drvdata(vfd, priv);
759 v4l2_ctrl_handler_init(&priv->ctrl_hdlr, 0);
763 EXPORT_SYMBOL_GPL(imx_media_capture_device_init);
765 void imx_media_capture_device_remove(struct imx_media_video_dev *vdev)
767 struct capture_priv *priv = to_capture_priv(vdev);
769 v4l2_ctrl_handler_free(&priv->ctrl_hdlr);
771 EXPORT_SYMBOL_GPL(imx_media_capture_device_remove);
773 MODULE_DESCRIPTION("i.MX5/6 v4l2 video capture interface driver");
774 MODULE_AUTHOR("Steve Longerbeam <steve_longerbeam@mentor.com>");
775 MODULE_LICENSE("GPL");