drivers/media/v4l2-core/v4l2-mem2mem.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Memory-to-memory device framework for Video for Linux 2 and videobuf.
   4  *
   5  * Helper functions for devices that use videobuf buffers for both their
   6  * source and destination.
   7  *
   8  * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
   9  * Pawel Osciak, <pawel@osciak.com>
  10  * Marek Szyprowski, <m.szyprowski@samsung.com>
  11  */
  12 #include <linux/module.h>
  13 #include <linux/sched.h>
  14 #include <linux/slab.h>
  15
  16 #include <media/media-device.h>
  17 #include <media/videobuf2-v4l2.h>
  18 #include <media/v4l2-mem2mem.h>
  19 #include <media/v4l2-dev.h>
  20 #include <media/v4l2-device.h>
  21 #include <media/v4l2-fh.h>
  22 #include <media/v4l2-event.h>
  23
  24 MODULE_DESCRIPTION("Mem to mem device framework for videobuf");
  25 MODULE_AUTHOR("Pawel Osciak, <pawel@osciak.com>");
  26 MODULE_LICENSE("GPL");
  27
  28 static bool debug;
  29 module_param(debug, bool, 0644);
  30
  31 #define dprintk(fmt, arg...)                                            \
  32         do {                                                            \
  33                 if (debug)                                              \
  34                         printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
  35         } while (0)
  36
  37
  38 /* Instance is already queued on the job_queue */
  39 #define TRANS_QUEUED            (1 << 0)
  40 /* Instance is currently running in hardware */
  41 #define TRANS_RUNNING           (1 << 1)
  42 /* Instance is currently aborting */
  43 #define TRANS_ABORT             (1 << 2)
  44
  45
  46 /* The job queue is not running new jobs */
  47 #define QUEUE_PAUSED            (1 << 0)
  48
  49
  50 /* Offset base for buffers on the destination queue - used to distinguish
  51  * between source and destination buffers when mmapping - they receive the same
  52  * offsets but for different queues */
  53 #define DST_QUEUE_OFF_BASE      (1 << 30)
  54
  55 enum v4l2_m2m_entity_type {
  56         MEM2MEM_ENT_TYPE_SOURCE,
  57         MEM2MEM_ENT_TYPE_SINK,
  58         MEM2MEM_ENT_TYPE_PROC
  59 };
  60
  61 static const char * const m2m_entity_name[] = {
  62         "source",
  63         "sink",
  64         "proc"
  65 };
  66
  67 /**
  68  * struct v4l2_m2m_dev - per-device context
  69  * @source:             &struct media_entity pointer with the source entity
  70  *                      Used only when the M2M device is registered via
  71  *                      v4l2_m2m_unregister_media_controller().
  72  * @source_pad:         &struct media_pad with the source pad.
  73  *                      Used only when the M2M device is registered via
  74  *                      v4l2_m2m_unregister_media_controller().
  75  * @sink:               &struct media_entity pointer with the sink entity
  76  *                      Used only when the M2M device is registered via
  77  *                      v4l2_m2m_unregister_media_controller().
  78  * @sink_pad:           &struct media_pad with the sink pad.
  79  *                      Used only when the M2M device is registered via
  80  *                      v4l2_m2m_unregister_media_controller().
  81  * @proc:               &struct media_entity pointer with the M2M device itself.
  82  * @proc_pads:          &struct media_pad with the @proc pads.
  83  *                      Used only when the M2M device is registered via
  84  *                      v4l2_m2m_unregister_media_controller().
  85  * @intf_devnode:       &struct media_intf devnode pointer with the interface
  86  *                      with controls the M2M device.
  87  * @curr_ctx:           currently running instance
  88  * @job_queue:          instances queued to run
  89  * @job_spinlock:       protects job_queue
  90  * @job_work:           worker to run queued jobs.
  91  * @job_queue_flags:    flags of the queue status, %QUEUE_PAUSED.
  92  * @m2m_ops:            driver callbacks
  93  */
  94 struct v4l2_m2m_dev {
  95         struct v4l2_m2m_ctx     *curr_ctx;
  96 #ifdef CONFIG_MEDIA_CONTROLLER
  97         struct media_entity     *source;
  98         struct media_pad        source_pad;
  99         struct media_entity     sink;
 100         struct media_pad        sink_pad;
 101         struct media_entity     proc;
 102         struct media_pad        proc_pads[2];
 103         struct media_intf_devnode *intf_devnode;
 104 #endif
 105
 106         struct list_head        job_queue;
 107         spinlock_t              job_spinlock;
 108         struct work_struct      job_work;
 109         unsigned long           job_queue_flags;
 110
 111         const struct v4l2_m2m_ops *m2m_ops;
 112 };
 113
 114 static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx,
 115                                                 enum v4l2_buf_type type)
 116 {
 117         if (V4L2_TYPE_IS_OUTPUT(type))
 118                 return &m2m_ctx->out_q_ctx;
 119         else
 120                 return &m2m_ctx->cap_q_ctx;
 121 }
 122
 123 struct vb2_queue *v4l2_m2m_get_vq(struct v4l2_m2m_ctx *m2m_ctx,
 124                                        enum v4l2_buf_type type)
 125 {
 126         struct v4l2_m2m_queue_ctx *q_ctx;
 127
 128         q_ctx = get_queue_ctx(m2m_ctx, type);
 129         if (!q_ctx)
 130                 return NULL;
 131
 132         return &q_ctx->q;
 133 }
 134 EXPORT_SYMBOL(v4l2_m2m_get_vq);
 135
 136 struct vb2_v4l2_buffer *v4l2_m2m_next_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 137 {
 138         struct v4l2_m2m_buffer *b;
 139         unsigned long flags;
 140
 141         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 142
 143         if (list_empty(&q_ctx->rdy_queue)) {
 144                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 145                 return NULL;
 146         }
 147
 148         b = list_first_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 149         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 150         return &b->vb;
 151 }
 152 EXPORT_SYMBOL_GPL(v4l2_m2m_next_buf);
 153
 154 struct vb2_v4l2_buffer *v4l2_m2m_last_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 155 {
 156         struct v4l2_m2m_buffer *b;
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 160
 161         if (list_empty(&q_ctx->rdy_queue)) {
 162                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 163                 return NULL;
 164         }
 165
 166         b = list_last_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 167         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 168         return &b->vb;
 169 }
 170 EXPORT_SYMBOL_GPL(v4l2_m2m_last_buf);
 171
 172 struct vb2_v4l2_buffer *v4l2_m2m_buf_remove(struct v4l2_m2m_queue_ctx *q_ctx)
 173 {
 174         struct v4l2_m2m_buffer *b;
 175         unsigned long flags;
 176
 177         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 178         if (list_empty(&q_ctx->rdy_queue)) {
 179                 spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 180                 return NULL;
 181         }
 182         b = list_first_entry(&q_ctx->rdy_queue, struct v4l2_m2m_buffer, list);
 183         list_del(&b->list);
 184         q_ctx->num_rdy--;
 185         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 186
 187         return &b->vb;
 188 }
 189 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove);
 190
 191 void v4l2_m2m_buf_remove_by_buf(struct v4l2_m2m_queue_ctx *q_ctx,
 192                                 struct vb2_v4l2_buffer *vbuf)
 193 {
 194         struct v4l2_m2m_buffer *b;
 195         unsigned long flags;
 196
 197         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 198         b = container_of(vbuf, struct v4l2_m2m_buffer, vb);
 199         list_del(&b->list);
 200         q_ctx->num_rdy--;
 201         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 202 }
 203 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove_by_buf);
 204
 205 struct vb2_v4l2_buffer *
 206 v4l2_m2m_buf_remove_by_idx(struct v4l2_m2m_queue_ctx *q_ctx, unsigned int idx)
 207
 208 {
 209         struct v4l2_m2m_buffer *b, *tmp;
 210         struct vb2_v4l2_buffer *ret = NULL;
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 214         list_for_each_entry_safe(b, tmp, &q_ctx->rdy_queue, list) {
 215                 if (b->vb.vb2_buf.index == idx) {
 216                         list_del(&b->list);
 217                         q_ctx->num_rdy--;
 218                         ret = &b->vb;
 219                         break;
 220                 }
 221         }
 222         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 223
 224         return ret;
 225 }
 226 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove_by_idx);
 227
 228 /*
 229  * Scheduling handlers
 230  */
 231
 232 void *v4l2_m2m_get_curr_priv(struct v4l2_m2m_dev *m2m_dev)
 233 {
 234         unsigned long flags;
 235         void *ret = NULL;
 236
 237         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 238         if (m2m_dev->curr_ctx)
 239                 ret = m2m_dev->curr_ctx->priv;
 240         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 241
 242         return ret;
 243 }
 244 EXPORT_SYMBOL(v4l2_m2m_get_curr_priv);
 245
 246 /**
 247  * v4l2_m2m_try_run() - select next job to perform and run it if possible
 248  * @m2m_dev: per-device context
 249  *
 250  * Get next transaction (if present) from the waiting jobs list and run it.
 251  *
 252  * Note that this function can run on a given v4l2_m2m_ctx context,
 253  * but call .device_run for another context.
 254  */
 255 static void v4l2_m2m_try_run(struct v4l2_m2m_dev *m2m_dev)
 256 {
 257         unsigned long flags;
 258
 259         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 260         if (NULL != m2m_dev->curr_ctx) {
 261                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 262                 dprintk("Another instance is running, won't run now\n");
 263                 return;
 264         }
 265
 266         if (list_empty(&m2m_dev->job_queue)) {
 267                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 268                 dprintk("No job pending\n");
 269                 return;
 270         }
 271
 272         if (m2m_dev->job_queue_flags & QUEUE_PAUSED) {
 273                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 274                 dprintk("Running new jobs is paused\n");
 275                 return;
 276         }
 277
 278         m2m_dev->curr_ctx = list_first_entry(&m2m_dev->job_queue,
 279                                    struct v4l2_m2m_ctx, queue);
 280         m2m_dev->curr_ctx->job_flags |= TRANS_RUNNING;
 281         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 282
 283         dprintk("Running job on m2m_ctx: %p\n", m2m_dev->curr_ctx);
 284         m2m_dev->m2m_ops->device_run(m2m_dev->curr_ctx->priv);
 285 }
 286
 287 /*
 288  * __v4l2_m2m_try_queue() - queue a job
 289  * @m2m_dev: m2m device
 290  * @m2m_ctx: m2m context
 291  *
 292  * Check if this context is ready to queue a job.
 293  *
 294  * This function can run in interrupt context.
 295  */
 296 static void __v4l2_m2m_try_queue(struct v4l2_m2m_dev *m2m_dev,
 297                                  struct v4l2_m2m_ctx *m2m_ctx)
 298 {
 299         unsigned long flags_job;
 300         struct vb2_v4l2_buffer *dst, *src;
 301
 302         dprintk("Trying to schedule a job for m2m_ctx: %p\n", m2m_ctx);
 303
 304         if (!m2m_ctx->out_q_ctx.q.streaming
 305             || !m2m_ctx->cap_q_ctx.q.streaming) {
 306                 dprintk("Streaming needs to be on for both queues\n");
 307                 return;
 308         }
 309
 310         spin_lock_irqsave(&m2m_dev->job_spinlock, flags_job);
 311
 312         /* If the context is aborted then don't schedule it */
 313         if (m2m_ctx->job_flags & TRANS_ABORT) {
 314                 dprintk("Aborted context\n");
 315                 goto job_unlock;
 316         }
 317
 318         if (m2m_ctx->job_flags & TRANS_QUEUED) {
 319                 dprintk("On job queue already\n");
 320                 goto job_unlock;
 321         }
 322
 323         src = v4l2_m2m_next_src_buf(m2m_ctx);
 324         dst = v4l2_m2m_next_dst_buf(m2m_ctx);
 325         if (!src && !m2m_ctx->out_q_ctx.buffered) {
 326                 dprintk("No input buffers available\n");
 327                 goto job_unlock;
 328         }
 329         if (!dst && !m2m_ctx->cap_q_ctx.buffered) {
 330                 dprintk("No output buffers available\n");
 331                 goto job_unlock;
 332         }
 333
 334         m2m_ctx->new_frame = true;
 335
 336         if (src && dst && dst->is_held &&
 337             dst->vb2_buf.copied_timestamp &&
 338             dst->vb2_buf.timestamp != src->vb2_buf.timestamp) {
 339                 dst->is_held = false;
 340                 v4l2_m2m_dst_buf_remove(m2m_ctx);
 341                 v4l2_m2m_buf_done(dst, VB2_BUF_STATE_DONE);
 342                 dst = v4l2_m2m_next_dst_buf(m2m_ctx);
 343
 344                 if (!dst && !m2m_ctx->cap_q_ctx.buffered) {
 345                         dprintk("No output buffers available after returning held buffer\n");
 346                         goto job_unlock;
 347                 }
 348         }
 349
 350         if (src && dst && (m2m_ctx->out_q_ctx.q.subsystem_flags &
 351                            VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF))
 352                 m2m_ctx->new_frame = !dst->vb2_buf.copied_timestamp ||
 353                         dst->vb2_buf.timestamp != src->vb2_buf.timestamp;
 354
 355         if (m2m_ctx->has_stopped) {
 356                 dprintk("Device has stopped\n");
 357                 goto job_unlock;
 358         }
 359
 360         if (m2m_dev->m2m_ops->job_ready
 361                 && (!m2m_dev->m2m_ops->job_ready(m2m_ctx->priv))) {
 362                 dprintk("Driver not ready\n");
 363                 goto job_unlock;
 364         }
 365
 366         list_add_tail(&m2m_ctx->queue, &m2m_dev->job_queue);
 367         m2m_ctx->job_flags |= TRANS_QUEUED;
 368
 369 job_unlock:
 370         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags_job);
 371 }
 372
 373 /**
 374  * v4l2_m2m_try_schedule() - schedule and possibly run a job for any context
 375  * @m2m_ctx: m2m context
 376  *
 377  * Check if this context is ready to queue a job. If suitable,
 378  * run the next queued job on the mem2mem device.
 379  *
 380  * This function shouldn't run in interrupt context.
 381  *
 382  * Note that v4l2_m2m_try_schedule() can schedule one job for this context,
 383  * and then run another job for another context.
 384  */
 385 void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx)
 386 {
 387         struct v4l2_m2m_dev *m2m_dev = m2m_ctx->m2m_dev;
 388
 389         __v4l2_m2m_try_queue(m2m_dev, m2m_ctx);
 390         v4l2_m2m_try_run(m2m_dev);
 391 }
 392 EXPORT_SYMBOL_GPL(v4l2_m2m_try_schedule);
 393
 394 /**
 395  * v4l2_m2m_device_run_work() - run pending jobs for the context
 396  * @work: Work structure used for scheduling the execution of this function.
 397  */
 398 static void v4l2_m2m_device_run_work(struct work_struct *work)
 399 {
 400         struct v4l2_m2m_dev *m2m_dev =
 401                 container_of(work, struct v4l2_m2m_dev, job_work);
 402
 403         v4l2_m2m_try_run(m2m_dev);
 404 }
 405
 406 /**
 407  * v4l2_m2m_cancel_job() - cancel pending jobs for the context
 408  * @m2m_ctx: m2m context with jobs to be canceled
 409  *
 410  * In case of streamoff or release called on any context,
 411  * 1] If the context is currently running, then abort job will be called
 412  * 2] If the context is queued, then the context will be removed from
 413  *    the job_queue
 414  */
 415 static void v4l2_m2m_cancel_job(struct v4l2_m2m_ctx *m2m_ctx)
 416 {
 417         struct v4l2_m2m_dev *m2m_dev;
 418         unsigned long flags;
 419
 420         m2m_dev = m2m_ctx->m2m_dev;
 421         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 422
 423         m2m_ctx->job_flags |= TRANS_ABORT;
 424         if (m2m_ctx->job_flags & TRANS_RUNNING) {
 425                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 426                 if (m2m_dev->m2m_ops->job_abort)
 427                         m2m_dev->m2m_ops->job_abort(m2m_ctx->priv);
 428                 dprintk("m2m_ctx %p running, will wait to complete\n", m2m_ctx);
 429                 wait_event(m2m_ctx->finished,
 430                                 !(m2m_ctx->job_flags & TRANS_RUNNING));
 431         } else if (m2m_ctx->job_flags & TRANS_QUEUED) {
 432                 list_del(&m2m_ctx->queue);
 433                 m2m_ctx->job_flags &= ~(TRANS_QUEUED | TRANS_RUNNING);
 434                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 435                 dprintk("m2m_ctx: %p had been on queue and was removed\n",
 436                         m2m_ctx);
 437         } else {
 438                 /* Do nothing, was not on queue/running */
 439                 spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 440         }
 441 }
 442
 443 /*
 444  * Schedule the next job, called from v4l2_m2m_job_finish() or
 445  * v4l2_m2m_buf_done_and_job_finish().
 446  */
 447 static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
 448                                        struct v4l2_m2m_ctx *m2m_ctx)
 449 {
 450         /*
 451          * This instance might have more buffers ready, but since we do not
 452          * allow more than one job on the job_queue per instance, each has
 453          * to be scheduled separately after the previous one finishes.
 454          */
 455         __v4l2_m2m_try_queue(m2m_dev, m2m_ctx);
 456
 457         /*
 458          * We might be running in atomic context,
 459          * but the job must be run in non-atomic context.
 460          */
 461         schedule_work(&m2m_dev->job_work);
 462 }
 463
 464 /*
 465  * Assumes job_spinlock is held, called from v4l2_m2m_job_finish() or
 466  * v4l2_m2m_buf_done_and_job_finish().
 467  */
 468 static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 469                                  struct v4l2_m2m_ctx *m2m_ctx)
 470 {
 471         if (!m2m_dev->curr_ctx || m2m_dev->curr_ctx != m2m_ctx) {
 472                 dprintk("Called by an instance not currently running\n");
 473                 return false;
 474         }
 475
 476         list_del(&m2m_dev->curr_ctx->queue);
 477         m2m_dev->curr_ctx->job_flags &= ~(TRANS_QUEUED | TRANS_RUNNING);
 478         wake_up(&m2m_dev->curr_ctx->finished);
 479         m2m_dev->curr_ctx = NULL;
 480         return true;
 481 }
 482
 483 void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 484                          struct v4l2_m2m_ctx *m2m_ctx)
 485 {
 486         unsigned long flags;
 487         bool schedule_next;
 488
 489         /*
 490          * This function should not be used for drivers that support
 491          * holding capture buffers. Those should use
 492          * v4l2_m2m_buf_done_and_job_finish() instead.
 493          */
 494         WARN_ON(m2m_ctx->out_q_ctx.q.subsystem_flags &
 495                 VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF);
 496         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 497         schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
 498         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 499
 500         if (schedule_next)
 501                 v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
 502 }
 503 EXPORT_SYMBOL(v4l2_m2m_job_finish);
 504
 505 void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
 506                                       struct v4l2_m2m_ctx *m2m_ctx,
 507                                       enum vb2_buffer_state state)
 508 {
 509         struct vb2_v4l2_buffer *src_buf, *dst_buf;
 510         bool schedule_next = false;
 511         unsigned long flags;
 512
 513         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 514         src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
 515         dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
 516
 517         if (WARN_ON(!src_buf || !dst_buf))
 518                 goto unlock;
 519         dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
 520         if (!dst_buf->is_held) {
 521                 v4l2_m2m_dst_buf_remove(m2m_ctx);
 522                 v4l2_m2m_buf_done(dst_buf, state);
 523         }
 524         /*
 525          * If the request API is being used, returning the OUTPUT
 526          * (src) buffer will wake-up any process waiting on the
 527          * request file descriptor.
 528          *
 529          * Therefore, return the CAPTURE (dst) buffer first,
 530          * to avoid signalling the request file descriptor
 531          * before the CAPTURE buffer is done.
 532          */
 533         v4l2_m2m_buf_done(src_buf, state);
 534         schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
 535 unlock:
 536         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 537
 538         if (schedule_next)
 539                 v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
 540 }
 541 EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
 542
 543 void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev)
 544 {
 545         unsigned long flags;
 546         struct v4l2_m2m_ctx *curr_ctx;
 547
 548         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 549         m2m_dev->job_queue_flags |= QUEUE_PAUSED;
 550         curr_ctx = m2m_dev->curr_ctx;
 551         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 552
 553         if (curr_ctx)
 554                 wait_event(curr_ctx->finished,
 555                            !(curr_ctx->job_flags & TRANS_RUNNING));
 556 }
 557 EXPORT_SYMBOL(v4l2_m2m_suspend);
 558
 559 void v4l2_m2m_resume(struct v4l2_m2m_dev *m2m_dev)
 560 {
 561         unsigned long flags;
 562
 563         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
 564         m2m_dev->job_queue_flags &= ~QUEUE_PAUSED;
 565         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
 566
 567         v4l2_m2m_try_run(m2m_dev);
 568 }
 569 EXPORT_SYMBOL(v4l2_m2m_resume);
 570
 571 int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 572                      struct v4l2_requestbuffers *reqbufs)
 573 {
 574         struct vb2_queue *vq;
 575         int ret;
 576
 577         vq = v4l2_m2m_get_vq(m2m_ctx, reqbufs->type);
 578         ret = vb2_reqbufs(vq, reqbufs);
 579         /* If count == 0, then the owner has released all buffers and he
 580            is no longer owner of the queue. Otherwise we have an owner. */
 581         if (ret == 0)
 582                 vq->owner = reqbufs->count ? file->private_data : NULL;
 583
 584         return ret;
 585 }
 586 EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs);
 587
 588 static void v4l2_m2m_adjust_mem_offset(struct vb2_queue *vq,
 589                                        struct v4l2_buffer *buf)
 590 {
 591         /* Adjust MMAP memory offsets for the CAPTURE queue */
 592         if (buf->memory == V4L2_MEMORY_MMAP && V4L2_TYPE_IS_CAPTURE(vq->type)) {
 593                 if (V4L2_TYPE_IS_MULTIPLANAR(vq->type)) {
 594                         unsigned int i;
 595
 596                         for (i = 0; i < buf->length; ++i)
 597                                 buf->m.planes[i].m.mem_offset
 598                                         += DST_QUEUE_OFF_BASE;
 599                 } else {
 600                         buf->m.offset += DST_QUEUE_OFF_BASE;
 601                 }
 602         }
 603 }
 604
 605 int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 606                       struct v4l2_buffer *buf)
 607 {
 608         struct vb2_queue *vq;
 609         int ret;
 610
 611         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 612         ret = vb2_querybuf(vq, buf);
 613         if (ret)
 614                 return ret;
 615
 616         /* Adjust MMAP memory offsets for the CAPTURE queue */
 617         v4l2_m2m_adjust_mem_offset(vq, buf);
 618
 619         return 0;
 620 }
 621 EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
 622
 623 /*
 624  * This will add the LAST flag and mark the buffer management
 625  * state as stopped.
 626  * This is called when the last capture buffer must be flagged as LAST
 627  * in draining mode from the encoder/decoder driver buf_queue() callback
 628  * or from v4l2_update_last_buf_state() when a capture buffer is available.
 629  */
 630 void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
 631                                struct vb2_v4l2_buffer *vbuf)
 632 {
 633         vbuf->flags |= V4L2_BUF_FLAG_LAST;
 634         vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
 635
 636         v4l2_m2m_mark_stopped(m2m_ctx);
 637 }
 638 EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
 639
 640 /* When stop command is issued, update buffer management state */
 641 static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
 642 {
 643         struct vb2_v4l2_buffer *next_dst_buf;
 644
 645         if (m2m_ctx->is_draining)
 646                 return -EBUSY;
 647
 648         if (m2m_ctx->has_stopped)
 649                 return 0;
 650
 651         m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
 652         m2m_ctx->is_draining = true;
 653
 654         /*
 655          * The processing of the last output buffer queued before
 656          * the STOP command is expected to mark the buffer management
 657          * state as stopped with v4l2_m2m_mark_stopped().
 658          */
 659         if (m2m_ctx->last_src_buf)
 660                 return 0;
 661
 662         /*
 663          * In case the output queue is empty, try to mark the last capture
 664          * buffer as LAST.
 665          */
 666         next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
 667         if (!next_dst_buf) {
 668                 /*
 669                  * Wait for the next queued one in encoder/decoder driver
 670                  * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
 671                  * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
 672                  * streaming.
 673                  */
 674                 m2m_ctx->next_buf_last = true;
 675                 return 0;
 676         }
 677
 678         v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
 679
 680         return 0;
 681 }
 682
 683 /*
 684  * Updates the encoding/decoding buffer management state, should
 685  * be called from encoder/decoder drivers start_streaming()
 686  */
 687 void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
 688                                            struct vb2_queue *q)
 689 {
 690         /* If start streaming again, untag the last output buffer */
 691         if (V4L2_TYPE_IS_OUTPUT(q->type))
 692                 m2m_ctx->last_src_buf = NULL;
 693 }
 694 EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
 695
 696 /*
 697  * Updates the encoding/decoding buffer management state, should
 698  * be called from encoder/decoder driver stop_streaming()
 699  */
 700 void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
 701                                           struct vb2_queue *q)
 702 {
 703         if (V4L2_TYPE_IS_OUTPUT(q->type)) {
 704                 /*
 705                  * If in draining state, either mark next dst buffer as
 706                  * done or flag next one to be marked as done either
 707                  * in encoder/decoder driver buf_queue() callback using
 708                  * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
 709                  * if encoder/decoder is not yet streaming
 710                  */
 711                 if (m2m_ctx->is_draining) {
 712                         struct vb2_v4l2_buffer *next_dst_buf;
 713
 714                         m2m_ctx->last_src_buf = NULL;
 715                         next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
 716                         if (!next_dst_buf)
 717                                 m2m_ctx->next_buf_last = true;
 718                         else
 719                                 v4l2_m2m_last_buffer_done(m2m_ctx,
 720                                                           next_dst_buf);
 721                 }
 722         } else {
 723                 v4l2_m2m_clear_state(m2m_ctx);
 724         }
 725 }
 726 EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
 727
 728 static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
 729                                          struct vb2_queue *q)
 730 {
 731         struct vb2_buffer *vb;
 732         struct vb2_v4l2_buffer *vbuf;
 733         unsigned int i;
 734
 735         if (WARN_ON(q->is_output))
 736                 return;
 737         if (list_empty(&q->queued_list))
 738                 return;
 739
 740         vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
 741         for (i = 0; i < vb->num_planes; i++)
 742                 vb2_set_plane_payload(vb, i, 0);
 743
 744         /*
 745          * Since the buffer hasn't been queued to the ready queue,
 746          * mark is active and owned before marking it LAST and DONE
 747          */
 748         vb->state = VB2_BUF_STATE_ACTIVE;
 749         atomic_inc(&q->owned_by_drv_count);
 750
 751         vbuf = to_vb2_v4l2_buffer(vb);
 752         vbuf->field = V4L2_FIELD_NONE;
 753
 754         v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
 755 }
 756
 757 int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 758                   struct v4l2_buffer *buf)
 759 {
 760         struct video_device *vdev = video_devdata(file);
 761         struct vb2_queue *vq;
 762         int ret;
 763
 764         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 765         if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
 766             (buf->flags & V4L2_BUF_FLAG_REQUEST_FD)) {
 767                 dprintk("%s: requests cannot be used with capture buffers\n",
 768                         __func__);
 769                 return -EPERM;
 770         }
 771
 772         ret = vb2_qbuf(vq, vdev->v4l2_dev->mdev, buf);
 773         if (ret)
 774                 return ret;
 775
 776         /* Adjust MMAP memory offsets for the CAPTURE queue */
 777         v4l2_m2m_adjust_mem_offset(vq, buf);
 778
 779         /*
 780          * If the capture queue is streaming, but streaming hasn't started
 781          * on the device, but was asked to stop, mark the previously queued
 782          * buffer as DONE with LAST flag since it won't be queued on the
 783          * device.
 784          */
 785         if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
 786             vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
 787            (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
 788                 v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
 789         else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
 790                 v4l2_m2m_try_schedule(m2m_ctx);
 791
 792         return 0;
 793 }
 794 EXPORT_SYMBOL_GPL(v4l2_m2m_qbuf);
 795
 796 int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 797                    struct v4l2_buffer *buf)
 798 {
 799         struct vb2_queue *vq;
 800         int ret;
 801
 802         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 803         ret = vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK);
 804         if (ret)
 805                 return ret;
 806
 807         /* Adjust MMAP memory offsets for the CAPTURE queue */
 808         v4l2_m2m_adjust_mem_offset(vq, buf);
 809
 810         return 0;
 811 }
 812 EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf);
 813
 814 int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 815                          struct v4l2_buffer *buf)
 816 {
 817         struct video_device *vdev = video_devdata(file);
 818         struct vb2_queue *vq;
 819         int ret;
 820
 821         vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
 822         ret = vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf);
 823         if (ret)
 824                 return ret;
 825
 826         /* Adjust MMAP memory offsets for the CAPTURE queue */
 827         v4l2_m2m_adjust_mem_offset(vq, buf);
 828
 829         return 0;
 830 }
 831 EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf);
 832
 833 int v4l2_m2m_create_bufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 834                          struct v4l2_create_buffers *create)
 835 {
 836         struct vb2_queue *vq;
 837
 838         vq = v4l2_m2m_get_vq(m2m_ctx, create->format.type);
 839         return vb2_create_bufs(vq, create);
 840 }
 841 EXPORT_SYMBOL_GPL(v4l2_m2m_create_bufs);
 842
 843 int v4l2_m2m_expbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 844                   struct v4l2_exportbuffer *eb)
 845 {
 846         struct vb2_queue *vq;
 847
 848         vq = v4l2_m2m_get_vq(m2m_ctx, eb->type);
 849         return vb2_expbuf(vq, eb);
 850 }
 851 EXPORT_SYMBOL_GPL(v4l2_m2m_expbuf);
 852
 853 int v4l2_m2m_streamon(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 854                       enum v4l2_buf_type type)
 855 {
 856         struct vb2_queue *vq;
 857         int ret;
 858
 859         vq = v4l2_m2m_get_vq(m2m_ctx, type);
 860         ret = vb2_streamon(vq, type);
 861         if (!ret)
 862                 v4l2_m2m_try_schedule(m2m_ctx);
 863
 864         return ret;
 865 }
 866 EXPORT_SYMBOL_GPL(v4l2_m2m_streamon);
 867
 868 int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 869                        enum v4l2_buf_type type)
 870 {
 871         struct v4l2_m2m_dev *m2m_dev;
 872         struct v4l2_m2m_queue_ctx *q_ctx;
 873         unsigned long flags_job, flags;
 874         int ret;
 875
 876         /* wait until the current context is dequeued from job_queue */
 877         v4l2_m2m_cancel_job(m2m_ctx);
 878
 879         q_ctx = get_queue_ctx(m2m_ctx, type);
 880         ret = vb2_streamoff(&q_ctx->q, type);
 881         if (ret)
 882                 return ret;
 883
 884         m2m_dev = m2m_ctx->m2m_dev;
 885         spin_lock_irqsave(&m2m_dev->job_spinlock, flags_job);
 886         /* We should not be scheduled anymore, since we're dropping a queue. */
 887         if (m2m_ctx->job_flags & TRANS_QUEUED)
 888                 list_del(&m2m_ctx->queue);
 889         m2m_ctx->job_flags = 0;
 890
 891         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
 892         /* Drop queue, since streamoff returns device to the same state as after
 893          * calling reqbufs. */
 894         INIT_LIST_HEAD(&q_ctx->rdy_queue);
 895         q_ctx->num_rdy = 0;
 896         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
 897
 898         if (m2m_dev->curr_ctx == m2m_ctx) {
 899                 m2m_dev->curr_ctx = NULL;
 900                 wake_up(&m2m_ctx->finished);
 901         }
 902         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags_job);
 903
 904         return 0;
 905 }
 906 EXPORT_SYMBOL_GPL(v4l2_m2m_streamoff);
 907
 908 static __poll_t v4l2_m2m_poll_for_data(struct file *file,
 909                                        struct v4l2_m2m_ctx *m2m_ctx,
 910                                        struct poll_table_struct *wait)
 911 {
 912         struct vb2_queue *src_q, *dst_q;
 913         __poll_t rc = 0;
 914         unsigned long flags;
 915
 916         src_q = v4l2_m2m_get_src_vq(m2m_ctx);
 917         dst_q = v4l2_m2m_get_dst_vq(m2m_ctx);
 918
 919         poll_wait(file, &src_q->done_wq, wait);
 920         poll_wait(file, &dst_q->done_wq, wait);
 921
 922         /*
 923          * There has to be at least one buffer queued on each queued_list, which
 924          * means either in driver already or waiting for driver to claim it
 925          * and start processing.
 926          */
 927         if ((!src_q->streaming || src_q->error ||
 928              list_empty(&src_q->queued_list)) &&
 929             (!dst_q->streaming || dst_q->error ||
 930              (list_empty(&dst_q->queued_list) && !dst_q->last_buffer_dequeued)))
 931                 return EPOLLERR;
 932
 933         spin_lock_irqsave(&src_q->done_lock, flags);
 934         if (!list_empty(&src_q->done_list))
 935                 rc |= EPOLLOUT | EPOLLWRNORM;
 936         spin_unlock_irqrestore(&src_q->done_lock, flags);
 937
 938         spin_lock_irqsave(&dst_q->done_lock, flags);
 939         /*
 940          * If the last buffer was dequeued from the capture queue, signal
 941          * userspace. DQBUF(CAPTURE) will return -EPIPE.
 942          */
 943         if (!list_empty(&dst_q->done_list) || dst_q->last_buffer_dequeued)
 944                 rc |= EPOLLIN | EPOLLRDNORM;
 945         spin_unlock_irqrestore(&dst_q->done_lock, flags);
 946
 947         return rc;
 948 }
 949
 950 __poll_t v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 951                        struct poll_table_struct *wait)
 952 {
 953         struct video_device *vfd = video_devdata(file);
 954         __poll_t req_events = poll_requested_events(wait);
 955         __poll_t rc = 0;
 956
 957         if (req_events & (EPOLLOUT | EPOLLWRNORM | EPOLLIN | EPOLLRDNORM))
 958                 rc = v4l2_m2m_poll_for_data(file, m2m_ctx, wait);
 959
 960         if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) {
 961                 struct v4l2_fh *fh = file->private_data;
 962
 963                 poll_wait(file, &fh->wait, wait);
 964                 if (v4l2_event_pending(fh))
 965                         rc |= EPOLLPRI;
 966         }
 967
 968         return rc;
 969 }
 970 EXPORT_SYMBOL_GPL(v4l2_m2m_poll);
 971
 972 int v4l2_m2m_mmap(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 973                          struct vm_area_struct *vma)
 974 {
 975         unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
 976         struct vb2_queue *vq;
 977
 978         if (offset < DST_QUEUE_OFF_BASE) {
 979                 vq = v4l2_m2m_get_src_vq(m2m_ctx);
 980         } else {
 981                 vq = v4l2_m2m_get_dst_vq(m2m_ctx);
 982                 vma->vm_pgoff -= (DST_QUEUE_OFF_BASE >> PAGE_SHIFT);
 983         }
 984
 985         return vb2_mmap(vq, vma);
 986 }
 987 EXPORT_SYMBOL(v4l2_m2m_mmap);
 988
 989 #if defined(CONFIG_MEDIA_CONTROLLER)
 990 void v4l2_m2m_unregister_media_controller(struct v4l2_m2m_dev *m2m_dev)
 991 {
 992         media_remove_intf_links(&m2m_dev->intf_devnode->intf);
 993         media_devnode_remove(m2m_dev->intf_devnode);
 994
 995         media_entity_remove_links(m2m_dev->source);
 996         media_entity_remove_links(&m2m_dev->sink);
 997         media_entity_remove_links(&m2m_dev->proc);
 998         media_device_unregister_entity(m2m_dev->source);
 999         media_device_unregister_entity(&m2m_dev->sink);
1000         media_device_unregister_entity(&m2m_dev->proc);
1001         kfree(m2m_dev->source->name);
1002         kfree(m2m_dev->sink.name);
1003         kfree(m2m_dev->proc.name);
1004 }
1005 EXPORT_SYMBOL_GPL(v4l2_m2m_unregister_media_controller);
1006
1007 static int v4l2_m2m_register_entity(struct media_device *mdev,
1008         struct v4l2_m2m_dev *m2m_dev, enum v4l2_m2m_entity_type type,
1009         struct video_device *vdev, int function)
1010 {
1011         struct media_entity *entity;
1012         struct media_pad *pads;
1013         char *name;
1014         unsigned int len;
1015         int num_pads;
1016         int ret;
1017
1018         switch (type) {
1019         case MEM2MEM_ENT_TYPE_SOURCE:
1020                 entity = m2m_dev->source;
1021                 pads = &m2m_dev->source_pad;
1022                 pads[0].flags = MEDIA_PAD_FL_SOURCE;
1023                 num_pads = 1;
1024                 break;
1025         case MEM2MEM_ENT_TYPE_SINK:
1026                 entity = &m2m_dev->sink;
1027                 pads = &m2m_dev->sink_pad;
1028                 pads[0].flags = MEDIA_PAD_FL_SINK;
1029                 num_pads = 1;
1030                 break;
1031         case MEM2MEM_ENT_TYPE_PROC:
1032                 entity = &m2m_dev->proc;
1033                 pads = m2m_dev->proc_pads;
1034                 pads[0].flags = MEDIA_PAD_FL_SINK;
1035                 pads[1].flags = MEDIA_PAD_FL_SOURCE;
1036                 num_pads = 2;
1037                 break;
1038         default:
1039                 return -EINVAL;
1040         }
1041
1042         entity->obj_type = MEDIA_ENTITY_TYPE_BASE;
1043         if (type != MEM2MEM_ENT_TYPE_PROC) {
1044                 entity->info.dev.major = VIDEO_MAJOR;
1045                 entity->info.dev.minor = vdev->minor;
1046         }
1047         len = strlen(vdev->name) + 2 + strlen(m2m_entity_name[type]);
1048         name = kmalloc(len, GFP_KERNEL);
1049         if (!name)
1050                 return -ENOMEM;
1051         snprintf(name, len, "%s-%s", vdev->name, m2m_entity_name[type]);
1052         entity->name = name;
1053         entity->function = function;
1054
1055         ret = media_entity_pads_init(entity, num_pads, pads);
1056         if (ret)
1057                 return ret;
1058         ret = media_device_register_entity(mdev, entity);
1059         if (ret)
1060                 return ret;
1061
1062         return 0;
1063 }
1064
1065 int v4l2_m2m_register_media_controller(struct v4l2_m2m_dev *m2m_dev,
1066                 struct video_device *vdev, int function)
1067 {
1068         struct media_device *mdev = vdev->v4l2_dev->mdev;
1069         struct media_link *link;
1070         int ret;
1071
1072         if (!mdev)
1073                 return 0;
1074
1075         /* A memory-to-memory device consists in two
1076          * DMA engine and one video processing entities.
1077          * The DMA engine entities are linked to a V4L interface
1078          */
1079
1080         /* Create the three entities with their pads */
1081         m2m_dev->source = &vdev->entity;
1082         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1083                         MEM2MEM_ENT_TYPE_SOURCE, vdev, MEDIA_ENT_F_IO_V4L);
1084         if (ret)
1085                 return ret;
1086         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1087                         MEM2MEM_ENT_TYPE_PROC, vdev, function);
1088         if (ret)
1089                 goto err_rel_entity0;
1090         ret = v4l2_m2m_register_entity(mdev, m2m_dev,
1091                         MEM2MEM_ENT_TYPE_SINK, vdev, MEDIA_ENT_F_IO_V4L);
1092         if (ret)
1093                 goto err_rel_entity1;
1094
1095         /* Connect the three entities */
1096         ret = media_create_pad_link(m2m_dev->source, 0, &m2m_dev->proc, 0,
1097                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1098         if (ret)
1099                 goto err_rel_entity2;
1100
1101         ret = media_create_pad_link(&m2m_dev->proc, 1, &m2m_dev->sink, 0,
1102                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1103         if (ret)
1104                 goto err_rm_links0;
1105
1106         /* Create video interface */
1107         m2m_dev->intf_devnode = media_devnode_create(mdev,
1108                         MEDIA_INTF_T_V4L_VIDEO, 0,
1109                         VIDEO_MAJOR, vdev->minor);
1110         if (!m2m_dev->intf_devnode) {
1111                 ret = -ENOMEM;
1112                 goto err_rm_links1;
1113         }
1114
1115         /* Connect the two DMA engines to the interface */
1116         link = media_create_intf_link(m2m_dev->source,
1117                         &m2m_dev->intf_devnode->intf,
1118                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1119         if (!link) {
1120                 ret = -ENOMEM;
1121                 goto err_rm_devnode;
1122         }
1123
1124         link = media_create_intf_link(&m2m_dev->sink,
1125                         &m2m_dev->intf_devnode->intf,
1126                         MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
1127         if (!link) {
1128                 ret = -ENOMEM;
1129                 goto err_rm_intf_link;
1130         }
1131         return 0;
1132
1133 err_rm_intf_link:
1134         media_remove_intf_links(&m2m_dev->intf_devnode->intf);
1135 err_rm_devnode:
1136         media_devnode_remove(m2m_dev->intf_devnode);
1137 err_rm_links1:
1138         media_entity_remove_links(&m2m_dev->sink);
1139 err_rm_links0:
1140         media_entity_remove_links(&m2m_dev->proc);
1141         media_entity_remove_links(m2m_dev->source);
1142 err_rel_entity2:
1143         media_device_unregister_entity(&m2m_dev->proc);
1144         kfree(m2m_dev->proc.name);
1145 err_rel_entity1:
1146         media_device_unregister_entity(&m2m_dev->sink);
1147         kfree(m2m_dev->sink.name);
1148 err_rel_entity0:
1149         media_device_unregister_entity(m2m_dev->source);
1150         kfree(m2m_dev->source->name);
1151         return ret;
1152         return 0;
1153 }
1154 EXPORT_SYMBOL_GPL(v4l2_m2m_register_media_controller);
1155 #endif
1156
1157 struct v4l2_m2m_dev *v4l2_m2m_init(const struct v4l2_m2m_ops *m2m_ops)
1158 {
1159         struct v4l2_m2m_dev *m2m_dev;
1160
1161         if (!m2m_ops || WARN_ON(!m2m_ops->device_run))
1162                 return ERR_PTR(-EINVAL);
1163
1164         m2m_dev = kzalloc(sizeof *m2m_dev, GFP_KERNEL);
1165         if (!m2m_dev)
1166                 return ERR_PTR(-ENOMEM);
1167
1168         m2m_dev->curr_ctx = NULL;
1169         m2m_dev->m2m_ops = m2m_ops;
1170         INIT_LIST_HEAD(&m2m_dev->job_queue);
1171         spin_lock_init(&m2m_dev->job_spinlock);
1172         INIT_WORK(&m2m_dev->job_work, v4l2_m2m_device_run_work);
1173
1174         return m2m_dev;
1175 }
1176 EXPORT_SYMBOL_GPL(v4l2_m2m_init);
1177
1178 void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev)
1179 {
1180         kfree(m2m_dev);
1181 }
1182 EXPORT_SYMBOL_GPL(v4l2_m2m_release);
1183
1184 struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(struct v4l2_m2m_dev *m2m_dev,
1185                 void *drv_priv,
1186                 int (*queue_init)(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq))
1187 {
1188         struct v4l2_m2m_ctx *m2m_ctx;
1189         struct v4l2_m2m_queue_ctx *out_q_ctx, *cap_q_ctx;
1190         int ret;
1191
1192         m2m_ctx = kzalloc(sizeof *m2m_ctx, GFP_KERNEL);
1193         if (!m2m_ctx)
1194                 return ERR_PTR(-ENOMEM);
1195
1196         m2m_ctx->priv = drv_priv;
1197         m2m_ctx->m2m_dev = m2m_dev;
1198         init_waitqueue_head(&m2m_ctx->finished);
1199
1200         out_q_ctx = &m2m_ctx->out_q_ctx;
1201         cap_q_ctx = &m2m_ctx->cap_q_ctx;
1202
1203         INIT_LIST_HEAD(&out_q_ctx->rdy_queue);
1204         INIT_LIST_HEAD(&cap_q_ctx->rdy_queue);
1205         spin_lock_init(&out_q_ctx->rdy_spinlock);
1206         spin_lock_init(&cap_q_ctx->rdy_spinlock);
1207
1208         INIT_LIST_HEAD(&m2m_ctx->queue);
1209
1210         ret = queue_init(drv_priv, &out_q_ctx->q, &cap_q_ctx->q);
1211
1212         if (ret)
1213                 goto err;
1214         /*
1215          * Both queues should use same the mutex to lock the m2m context.
1216          * This lock is used in some v4l2_m2m_* helpers.
1217          */
1218         if (WARN_ON(out_q_ctx->q.lock != cap_q_ctx->q.lock)) {
1219                 ret = -EINVAL;
1220                 goto err;
1221         }
1222         m2m_ctx->q_lock = out_q_ctx->q.lock;
1223
1224         return m2m_ctx;
1225 err:
1226         kfree(m2m_ctx);
1227         return ERR_PTR(ret);
1228 }
1229 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_init);
1230
1231 void v4l2_m2m_ctx_release(struct v4l2_m2m_ctx *m2m_ctx)
1232 {
1233         /* wait until the current context is dequeued from job_queue */
1234         v4l2_m2m_cancel_job(m2m_ctx);
1235
1236         vb2_queue_release(&m2m_ctx->cap_q_ctx.q);
1237         vb2_queue_release(&m2m_ctx->out_q_ctx.q);
1238
1239         kfree(m2m_ctx);
1240 }
1241 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_release);
1242
1243 void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *m2m_ctx,
1244                 struct vb2_v4l2_buffer *vbuf)
1245 {
1246         struct v4l2_m2m_buffer *b = container_of(vbuf,
1247                                 struct v4l2_m2m_buffer, vb);
1248         struct v4l2_m2m_queue_ctx *q_ctx;
1249         unsigned long flags;
1250
1251         q_ctx = get_queue_ctx(m2m_ctx, vbuf->vb2_buf.vb2_queue->type);
1252         if (!q_ctx)
1253                 return;
1254
1255         spin_lock_irqsave(&q_ctx->rdy_spinlock, flags);
1256         list_add_tail(&b->list, &q_ctx->rdy_queue);
1257         q_ctx->num_rdy++;
1258         spin_unlock_irqrestore(&q_ctx->rdy_spinlock, flags);
1259 }
1260 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_queue);
1261
1262 void v4l2_m2m_buf_copy_metadata(const struct vb2_v4l2_buffer *out_vb,
1263                                 struct vb2_v4l2_buffer *cap_vb,
1264                                 bool copy_frame_flags)
1265 {
1266         u32 mask = V4L2_BUF_FLAG_TIMECODE | V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
1267
1268         if (copy_frame_flags)
1269                 mask |= V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME |
1270                         V4L2_BUF_FLAG_BFRAME;
1271
1272         cap_vb->vb2_buf.timestamp = out_vb->vb2_buf.timestamp;
1273
1274         if (out_vb->flags & V4L2_BUF_FLAG_TIMECODE)
1275                 cap_vb->timecode = out_vb->timecode;
1276         cap_vb->field = out_vb->field;
1277         cap_vb->flags &= ~mask;
1278         cap_vb->flags |= out_vb->flags & mask;
1279         cap_vb->vb2_buf.copied_timestamp = 1;
1280 }
1281 EXPORT_SYMBOL_GPL(v4l2_m2m_buf_copy_metadata);
1282
1283 void v4l2_m2m_request_queue(struct media_request *req)
1284 {
1285         struct media_request_object *obj, *obj_safe;
1286         struct v4l2_m2m_ctx *m2m_ctx = NULL;
1287
1288         /*
1289          * Queue all objects. Note that buffer objects are at the end of the
1290          * objects list, after all other object types. Once buffer objects
1291          * are queued, the driver might delete them immediately (if the driver
1292          * processes the buffer at once), so we have to use
1293          * list_for_each_entry_safe() to handle the case where the object we
1294          * queue is deleted.
1295          */
1296         list_for_each_entry_safe(obj, obj_safe, &req->objects, list) {
1297                 struct v4l2_m2m_ctx *m2m_ctx_obj;
1298                 struct vb2_buffer *vb;
1299
1300                 if (!obj->ops->queue)
1301                         continue;
1302
1303                 if (vb2_request_object_is_buffer(obj)) {
1304                         /* Sanity checks */
1305                         vb = container_of(obj, struct vb2_buffer, req_obj);
1306                         WARN_ON(!V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type));
1307                         m2m_ctx_obj = container_of(vb->vb2_queue,
1308                                                    struct v4l2_m2m_ctx,
1309                                                    out_q_ctx.q);
1310                         WARN_ON(m2m_ctx && m2m_ctx_obj != m2m_ctx);
1311                         m2m_ctx = m2m_ctx_obj;
1312                 }
1313
1314                 /*
1315                  * The buffer we queue here can in theory be immediately
1316                  * unbound, hence the use of list_for_each_entry_safe()
1317                  * above and why we call the queue op last.
1318                  */
1319                 obj->ops->queue(obj);
1320         }
1321
1322         WARN_ON(!m2m_ctx);
1323
1324         if (m2m_ctx)
1325                 v4l2_m2m_try_schedule(m2m_ctx);
1326 }
1327 EXPORT_SYMBOL_GPL(v4l2_m2m_request_queue);
1328
1329 /* Videobuf2 ioctl helpers */
1330
1331 int v4l2_m2m_ioctl_reqbufs(struct file *file, void *priv,
1332                                 struct v4l2_requestbuffers *rb)
1333 {
1334         struct v4l2_fh *fh = file->private_data;
1335
1336         return v4l2_m2m_reqbufs(file, fh->m2m_ctx, rb);
1337 }
1338 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_reqbufs);
1339
1340 int v4l2_m2m_ioctl_create_bufs(struct file *file, void *priv,
1341                                 struct v4l2_create_buffers *create)
1342 {
1343         struct v4l2_fh *fh = file->private_data;
1344
1345         return v4l2_m2m_create_bufs(file, fh->m2m_ctx, create);
1346 }
1347 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_create_bufs);
1348
1349 int v4l2_m2m_ioctl_querybuf(struct file *file, void *priv,
1350                                 struct v4l2_buffer *buf)
1351 {
1352         struct v4l2_fh *fh = file->private_data;
1353
1354         return v4l2_m2m_querybuf(file, fh->m2m_ctx, buf);
1355 }
1356 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_querybuf);
1357
1358 int v4l2_m2m_ioctl_qbuf(struct file *file, void *priv,
1359                                 struct v4l2_buffer *buf)
1360 {
1361         struct v4l2_fh *fh = file->private_data;
1362
1363         return v4l2_m2m_qbuf(file, fh->m2m_ctx, buf);
1364 }
1365 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_qbuf);
1366
1367 int v4l2_m2m_ioctl_dqbuf(struct file *file, void *priv,
1368                                 struct v4l2_buffer *buf)
1369 {
1370         struct v4l2_fh *fh = file->private_data;
1371
1372         return v4l2_m2m_dqbuf(file, fh->m2m_ctx, buf);
1373 }
1374 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_dqbuf);
1375
1376 int v4l2_m2m_ioctl_prepare_buf(struct file *file, void *priv,
1377                                struct v4l2_buffer *buf)
1378 {
1379         struct v4l2_fh *fh = file->private_data;
1380
1381         return v4l2_m2m_prepare_buf(file, fh->m2m_ctx, buf);
1382 }
1383 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_prepare_buf);
1384
1385 int v4l2_m2m_ioctl_expbuf(struct file *file, void *priv,
1386                                 struct v4l2_exportbuffer *eb)
1387 {
1388         struct v4l2_fh *fh = file->private_data;
1389
1390         return v4l2_m2m_expbuf(file, fh->m2m_ctx, eb);
1391 }
1392 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_expbuf);
1393
1394 int v4l2_m2m_ioctl_streamon(struct file *file, void *priv,
1395                                 enum v4l2_buf_type type)
1396 {
1397         struct v4l2_fh *fh = file->private_data;
1398
1399         return v4l2_m2m_streamon(file, fh->m2m_ctx, type);
1400 }
1401 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamon);
1402
1403 int v4l2_m2m_ioctl_streamoff(struct file *file, void *priv,
1404                                 enum v4l2_buf_type type)
1405 {
1406         struct v4l2_fh *fh = file->private_data;
1407
1408         return v4l2_m2m_streamoff(file, fh->m2m_ctx, type);
1409 }
1410 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamoff);
1411
1412 int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
1413                                    struct v4l2_encoder_cmd *ec)
1414 {
1415         if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
1416                 return -EINVAL;
1417
1418         ec->flags = 0;
1419         return 0;
1420 }
1421 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
1422
1423 int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
1424                                    struct v4l2_decoder_cmd *dc)
1425 {
1426         if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
1427                 return -EINVAL;
1428
1429         dc->flags = 0;
1430
1431         if (dc->cmd == V4L2_DEC_CMD_STOP) {
1432                 dc->stop.pts = 0;
1433         } else if (dc->cmd == V4L2_DEC_CMD_START) {
1434                 dc->start.speed = 0;
1435                 dc->start.format = V4L2_DEC_START_FMT_NONE;
1436         }
1437         return 0;
1438 }
1439 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
1440
1441 /*
1442  * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
1443  * Should be called from the encoder driver encoder_cmd() callback
1444  */
1445 int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
1446                          struct v4l2_encoder_cmd *ec)
1447 {
1448         if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
1449                 return -EINVAL;
1450
1451         if (ec->cmd == V4L2_ENC_CMD_STOP)
1452                 return v4l2_update_last_buf_state(m2m_ctx);
1453
1454         if (m2m_ctx->is_draining)
1455                 return -EBUSY;
1456
1457         if (m2m_ctx->has_stopped)
1458                 m2m_ctx->has_stopped = false;
1459
1460         return 0;
1461 }
1462 EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
1463
1464 /*
1465  * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
1466  * Should be called from the decoder driver decoder_cmd() callback
1467  */
1468 int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
1469                          struct v4l2_decoder_cmd *dc)
1470 {
1471         if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
1472                 return -EINVAL;
1473
1474         if (dc->cmd == V4L2_DEC_CMD_STOP)
1475                 return v4l2_update_last_buf_state(m2m_ctx);
1476
1477         if (m2m_ctx->is_draining)
1478                 return -EBUSY;
1479
1480         if (m2m_ctx->has_stopped)
1481                 m2m_ctx->has_stopped = false;
1482
1483         return 0;
1484 }
1485 EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
1486
1487 int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
1488                                struct v4l2_encoder_cmd *ec)
1489 {
1490         struct v4l2_fh *fh = file->private_data;
1491
1492         return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
1493 }
1494 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
1495
1496 int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
1497                                struct v4l2_decoder_cmd *dc)
1498 {
1499         struct v4l2_fh *fh = file->private_data;
1500
1501         return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
1502 }
1503 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
1504
1505 int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
1506                                              struct v4l2_decoder_cmd *dc)
1507 {
1508         if (dc->cmd != V4L2_DEC_CMD_FLUSH)
1509                 return -EINVAL;
1510
1511         dc->flags = 0;
1512
1513         return 0;
1514 }
1515 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
1516
1517 int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
1518                                          struct v4l2_decoder_cmd *dc)
1519 {
1520         struct v4l2_fh *fh = file->private_data;
1521         struct vb2_v4l2_buffer *out_vb, *cap_vb;
1522         struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
1523         unsigned long flags;
1524         int ret;
1525
1526         ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
1527         if (ret < 0)
1528                 return ret;
1529
1530         spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
1531         out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
1532         cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
1533
1534         /*
1535          * If there is an out buffer pending, then clear any HOLD flag.
1536          *
1537          * By clearing this flag we ensure that when this output
1538          * buffer is processed any held capture buffer will be released.
1539          */
1540         if (out_vb) {
1541                 out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
1542         } else if (cap_vb && cap_vb->is_held) {
1543                 /*
1544                  * If there were no output buffers, but there is a
1545                  * capture buffer that is held, then release that
1546                  * buffer.
1547                  */
1548                 cap_vb->is_held = false;
1549                 v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
1550                 v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
1551         }
1552         spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
1553
1554         return 0;
1555 }
1556 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
1557
1558 /*
1559  * v4l2_file_operations helpers. It is assumed here same lock is used
1560  * for the output and the capture buffer queue.
1561  */
1562
1563 int v4l2_m2m_fop_mmap(struct file *file, struct vm_area_struct *vma)
1564 {
1565         struct v4l2_fh *fh = file->private_data;
1566
1567         return v4l2_m2m_mmap(file, fh->m2m_ctx, vma);
1568 }
1569 EXPORT_SYMBOL_GPL(v4l2_m2m_fop_mmap);
1570
1571 __poll_t v4l2_m2m_fop_poll(struct file *file, poll_table *wait)
1572 {
1573         struct v4l2_fh *fh = file->private_data;
1574         struct v4l2_m2m_ctx *m2m_ctx = fh->m2m_ctx;
1575         __poll_t ret;
1576
1577         if (m2m_ctx->q_lock)
1578                 mutex_lock(m2m_ctx->q_lock);
1579
1580         ret = v4l2_m2m_poll(file, m2m_ctx, wait);
1581
1582         if (m2m_ctx->q_lock)
1583                 mutex_unlock(m2m_ctx->q_lock);
1584
1585         return ret;
1586 }
1587 EXPORT_SYMBOL_GPL(v4l2_m2m_fop_poll);
1588