drivers/media/platform/coda/coda-bit.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Coda multi-standard codec IP - BIT processor functions
   4  *
   5  * Copyright (C) 2012 Vista Silicon S.L.
   6  *    Javier Martin, <javier.martin@vista-silicon.com>
   7  *    Xavier Duret
   8  * Copyright (C) 2012-2014 Philipp Zabel, Pengutronix
   9  */
  10
  11 #include <linux/clk.h>
  12 #include <linux/irqreturn.h>
  13 #include <linux/kernel.h>
  14 #include <linux/log2.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/reset.h>
  17 #include <linux/slab.h>
  18 #include <linux/videodev2.h>
  19
  20 #include <media/v4l2-common.h>
  21 #include <media/v4l2-ctrls.h>
  22 #include <media/v4l2-fh.h>
  23 #include <media/v4l2-mem2mem.h>
  24 #include <media/videobuf2-v4l2.h>
  25 #include <media/videobuf2-dma-contig.h>
  26 #include <media/videobuf2-vmalloc.h>
  27
  28 #include "coda.h"
  29 #include "imx-vdoa.h"
  30 #define CREATE_TRACE_POINTS
  31 #include "trace.h"
  32
  33 #define CODA_PARA_BUF_SIZE      (10 * 1024)
  34 #define CODA7_PS_BUF_SIZE       0x28000
  35 #define CODA9_PS_SAVE_SIZE      (512 * 1024)
  36
  37 #define CODA_DEFAULT_GAMMA      4096
  38 #define CODA9_DEFAULT_GAMMA     24576   /* 0.75 * 32768 */
  39
  40 static void coda_free_bitstream_buffer(struct coda_ctx *ctx);
  41
  42 static inline int coda_is_initialized(struct coda_dev *dev)
  43 {
  44         return coda_read(dev, CODA_REG_BIT_CUR_PC) != 0;
  45 }
  46
  47 static inline unsigned long coda_isbusy(struct coda_dev *dev)
  48 {
  49         return coda_read(dev, CODA_REG_BIT_BUSY);
  50 }
  51
  52 static int coda_wait_timeout(struct coda_dev *dev)
  53 {
  54         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
  55
  56         while (coda_isbusy(dev)) {
  57                 if (time_after(jiffies, timeout))
  58                         return -ETIMEDOUT;
  59         }
  60         return 0;
  61 }
  62
  63 static void coda_command_async(struct coda_ctx *ctx, int cmd)
  64 {
  65         struct coda_dev *dev = ctx->dev;
  66
  67         if (dev->devtype->product == CODA_HX4 ||
  68             dev->devtype->product == CODA_7541 ||
  69             dev->devtype->product == CODA_960) {
  70                 /* Restore context related registers to CODA */
  71                 coda_write(dev, ctx->bit_stream_param,
  72                                 CODA_REG_BIT_BIT_STREAM_PARAM);
  73                 coda_write(dev, ctx->frm_dis_flg,
  74                                 CODA_REG_BIT_FRM_DIS_FLG(ctx->reg_idx));
  75                 coda_write(dev, ctx->frame_mem_ctrl,
  76                                 CODA_REG_BIT_FRAME_MEM_CTRL);
  77                 coda_write(dev, ctx->workbuf.paddr, CODA_REG_BIT_WORK_BUF_ADDR);
  78         }
  79
  80         if (dev->devtype->product == CODA_960) {
  81                 coda_write(dev, 1, CODA9_GDI_WPROT_ERR_CLR);
  82                 coda_write(dev, 0, CODA9_GDI_WPROT_RGN_EN);
  83         }
  84
  85         coda_write(dev, CODA_REG_BIT_BUSY_FLAG, CODA_REG_BIT_BUSY);
  86
  87         coda_write(dev, ctx->idx, CODA_REG_BIT_RUN_INDEX);
  88         coda_write(dev, ctx->params.codec_mode, CODA_REG_BIT_RUN_COD_STD);
  89         coda_write(dev, ctx->params.codec_mode_aux, CODA7_REG_BIT_RUN_AUX_STD);
  90
  91         trace_coda_bit_run(ctx, cmd);
  92
  93         coda_write(dev, cmd, CODA_REG_BIT_RUN_COMMAND);
  94 }
  95
  96 static int coda_command_sync(struct coda_ctx *ctx, int cmd)
  97 {
  98         struct coda_dev *dev = ctx->dev;
  99         int ret;
 100
 101         lockdep_assert_held(&dev->coda_mutex);
 102
 103         coda_command_async(ctx, cmd);
 104         ret = coda_wait_timeout(dev);
 105         trace_coda_bit_done(ctx);
 106
 107         return ret;
 108 }
 109
 110 int coda_hw_reset(struct coda_ctx *ctx)
 111 {
 112         struct coda_dev *dev = ctx->dev;
 113         unsigned long timeout;
 114         unsigned int idx;
 115         int ret;
 116
 117         lockdep_assert_held(&dev->coda_mutex);
 118
 119         if (!dev->rstc)
 120                 return -ENOENT;
 121
 122         idx = coda_read(dev, CODA_REG_BIT_RUN_INDEX);
 123
 124         if (dev->devtype->product == CODA_960) {
 125                 timeout = jiffies + msecs_to_jiffies(100);
 126                 coda_write(dev, 0x11, CODA9_GDI_BUS_CTRL);
 127                 while (coda_read(dev, CODA9_GDI_BUS_STATUS) != 0x77) {
 128                         if (time_after(jiffies, timeout))
 129                                 return -ETIME;
 130                         cpu_relax();
 131                 }
 132         }
 133
 134         ret = reset_control_reset(dev->rstc);
 135         if (ret < 0)
 136                 return ret;
 137
 138         if (dev->devtype->product == CODA_960)
 139                 coda_write(dev, 0x00, CODA9_GDI_BUS_CTRL);
 140         coda_write(dev, CODA_REG_BIT_BUSY_FLAG, CODA_REG_BIT_BUSY);
 141         coda_write(dev, CODA_REG_RUN_ENABLE, CODA_REG_BIT_CODE_RUN);
 142         ret = coda_wait_timeout(dev);
 143         coda_write(dev, idx, CODA_REG_BIT_RUN_INDEX);
 144
 145         return ret;
 146 }
 147
 148 static void coda_kfifo_sync_from_device(struct coda_ctx *ctx)
 149 {
 150         struct __kfifo *kfifo = &ctx->bitstream_fifo.kfifo;
 151         struct coda_dev *dev = ctx->dev;
 152         u32 rd_ptr;
 153
 154         rd_ptr = coda_read(dev, CODA_REG_BIT_RD_PTR(ctx->reg_idx));
 155         kfifo->out = (kfifo->in & ~kfifo->mask) |
 156                       (rd_ptr - ctx->bitstream.paddr);
 157         if (kfifo->out > kfifo->in)
 158                 kfifo->out -= kfifo->mask + 1;
 159 }
 160
 161 static void coda_kfifo_sync_to_device_full(struct coda_ctx *ctx)
 162 {
 163         struct __kfifo *kfifo = &ctx->bitstream_fifo.kfifo;
 164         struct coda_dev *dev = ctx->dev;
 165         u32 rd_ptr, wr_ptr;
 166
 167         rd_ptr = ctx->bitstream.paddr + (kfifo->out & kfifo->mask);
 168         coda_write(dev, rd_ptr, CODA_REG_BIT_RD_PTR(ctx->reg_idx));
 169         wr_ptr = ctx->bitstream.paddr + (kfifo->in & kfifo->mask);
 170         coda_write(dev, wr_ptr, CODA_REG_BIT_WR_PTR(ctx->reg_idx));
 171 }
 172
 173 static void coda_kfifo_sync_to_device_write(struct coda_ctx *ctx)
 174 {
 175         struct __kfifo *kfifo = &ctx->bitstream_fifo.kfifo;
 176         struct coda_dev *dev = ctx->dev;
 177         u32 wr_ptr;
 178
 179         wr_ptr = ctx->bitstream.paddr + (kfifo->in & kfifo->mask);
 180         coda_write(dev, wr_ptr, CODA_REG_BIT_WR_PTR(ctx->reg_idx));
 181 }
 182
 183 static int coda_h264_bitstream_pad(struct coda_ctx *ctx, u32 size)
 184 {
 185         unsigned char *buf;
 186         u32 n;
 187
 188         if (size < 6)
 189                 size = 6;
 190
 191         buf = kmalloc(size, GFP_KERNEL);
 192         if (!buf)
 193                 return -ENOMEM;
 194
 195         coda_h264_filler_nal(size, buf);
 196         n = kfifo_in(&ctx->bitstream_fifo, buf, size);
 197         kfree(buf);
 198
 199         return (n < size) ? -ENOSPC : 0;
 200 }
 201
 202 int coda_bitstream_flush(struct coda_ctx *ctx)
 203 {
 204         int ret;
 205
 206         if (ctx->inst_type != CODA_INST_DECODER || !ctx->use_bit)
 207                 return 0;
 208
 209         ret = coda_command_sync(ctx, CODA_COMMAND_DEC_BUF_FLUSH);
 210         if (ret < 0) {
 211                 v4l2_err(&ctx->dev->v4l2_dev, "failed to flush bitstream\n");
 212                 return ret;
 213         }
 214
 215         kfifo_init(&ctx->bitstream_fifo, ctx->bitstream.vaddr,
 216                    ctx->bitstream.size);
 217         coda_kfifo_sync_to_device_full(ctx);
 218
 219         return 0;
 220 }
 221
 222 static int coda_bitstream_queue(struct coda_ctx *ctx, const u8 *buf, u32 size)
 223 {
 224         u32 n = kfifo_in(&ctx->bitstream_fifo, buf, size);
 225
 226         return (n < size) ? -ENOSPC : 0;
 227 }
 228
 229 static u32 coda_buffer_parse_headers(struct coda_ctx *ctx,
 230                                      struct vb2_v4l2_buffer *src_buf,
 231                                      u32 payload)
 232 {
 233         u8 *vaddr = vb2_plane_vaddr(&src_buf->vb2_buf, 0);
 234         u32 size = 0;
 235
 236         switch (ctx->codec->src_fourcc) {
 237         case V4L2_PIX_FMT_MPEG2:
 238                 size = coda_mpeg2_parse_headers(ctx, vaddr, payload);
 239                 break;
 240         case V4L2_PIX_FMT_MPEG4:
 241                 size = coda_mpeg4_parse_headers(ctx, vaddr, payload);
 242                 break;
 243         default:
 244                 break;
 245         }
 246
 247         return size;
 248 }
 249
 250 static bool coda_bitstream_try_queue(struct coda_ctx *ctx,
 251                                      struct vb2_v4l2_buffer *src_buf)
 252 {
 253         unsigned long payload = vb2_get_plane_payload(&src_buf->vb2_buf, 0);
 254         u8 *vaddr = vb2_plane_vaddr(&src_buf->vb2_buf, 0);
 255         int ret;
 256         int i;
 257
 258         if (coda_get_bitstream_payload(ctx) + payload + 512 >=
 259             ctx->bitstream.size)
 260                 return false;
 261
 262         if (!vaddr) {
 263                 v4l2_err(&ctx->dev->v4l2_dev, "trying to queue empty buffer\n");
 264                 return true;
 265         }
 266
 267         if (ctx->qsequence == 0 && payload < 512) {
 268                 /*
 269                  * Add padding after the first buffer, if it is too small to be
 270                  * fetched by the CODA, by repeating the headers. Without
 271                  * repeated headers, or the first frame already queued, decoder
 272                  * sequence initialization fails with error code 0x2000 on i.MX6
 273                  * or error code 0x1 on i.MX51.
 274                  */
 275                 u32 header_size = coda_buffer_parse_headers(ctx, src_buf,
 276                                                             payload);
 277
 278                 if (header_size) {
 279                         coda_dbg(1, ctx, "pad with %u-byte header\n",
 280                                  header_size);
 281                         for (i = payload; i < 512; i += header_size) {
 282                                 ret = coda_bitstream_queue(ctx, vaddr,
 283                                                            header_size);
 284                                 if (ret < 0) {
 285                                         v4l2_err(&ctx->dev->v4l2_dev,
 286                                                  "bitstream buffer overflow\n");
 287                                         return false;
 288                                 }
 289                                 if (ctx->dev->devtype->product == CODA_960)
 290                                         break;
 291                         }
 292                 } else {
 293                         coda_dbg(1, ctx,
 294                                  "could not parse header, sequence initialization might fail\n");
 295                 }
 296         }
 297
 298         /* Add padding before the first buffer, if it is too small */
 299         if (ctx->qsequence == 0 && payload < 512 &&
 300             ctx->codec->src_fourcc == V4L2_PIX_FMT_H264)
 301                 coda_h264_bitstream_pad(ctx, 512 - payload);
 302
 303         ret = coda_bitstream_queue(ctx, vaddr, payload);
 304         if (ret < 0) {
 305                 v4l2_err(&ctx->dev->v4l2_dev, "bitstream buffer overflow\n");
 306                 return false;
 307         }
 308
 309         src_buf->sequence = ctx->qsequence++;
 310
 311         /* Sync read pointer to device */
 312         if (ctx == v4l2_m2m_get_curr_priv(ctx->dev->m2m_dev))
 313                 coda_kfifo_sync_to_device_write(ctx);
 314
 315         /* Set the stream-end flag after the last buffer is queued */
 316         if (src_buf->flags & V4L2_BUF_FLAG_LAST)
 317                 coda_bit_stream_end_flag(ctx);
 318         ctx->hold = false;
 319
 320         return true;
 321 }
 322
 323 void coda_fill_bitstream(struct coda_ctx *ctx, struct list_head *buffer_list)
 324 {
 325         struct vb2_v4l2_buffer *src_buf;
 326         struct coda_buffer_meta *meta;
 327         u32 start;
 328
 329         if (ctx->bit_stream_param & CODA_BIT_STREAM_END_FLAG)
 330                 return;
 331
 332         while (v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx) > 0) {
 333                 /*
 334                  * Only queue two JPEGs into the bitstream buffer to keep
 335                  * latency low. We need at least one complete buffer and the
 336                  * header of another buffer (for prescan) in the bitstream.
 337                  */
 338                 if (ctx->codec->src_fourcc == V4L2_PIX_FMT_JPEG &&
 339                     ctx->num_metas > 1)
 340                         break;
 341
 342                 if (ctx->num_internal_frames &&
 343                     ctx->num_metas >= ctx->num_internal_frames) {
 344                         meta = list_first_entry(&ctx->buffer_meta_list,
 345                                                 struct coda_buffer_meta, list);
 346
 347                         /*
 348                          * If we managed to fill in at least a full reorder
 349                          * window of buffers (num_internal_frames is a
 350                          * conservative estimate for this) and the bitstream
 351                          * prefetcher has at least 2 256 bytes periods beyond
 352                          * the first buffer to fetch, we can safely stop queuing
 353                          * in order to limit the decoder drain latency.
 354                          */
 355                         if (coda_bitstream_can_fetch_past(ctx, meta->end))
 356                                 break;
 357                 }
 358
 359                 src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 360
 361                 /* Drop frames that do not start/end with a SOI/EOI markers */
 362                 if (ctx->codec->src_fourcc == V4L2_PIX_FMT_JPEG &&
 363                     !coda_jpeg_check_buffer(ctx, &src_buf->vb2_buf)) {
 364                         v4l2_err(&ctx->dev->v4l2_dev,
 365                                  "dropping invalid JPEG frame %d\n",
 366                                  ctx->qsequence);
 367                         src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
 368                         if (buffer_list) {
 369                                 struct v4l2_m2m_buffer *m2m_buf;
 370
 371                                 m2m_buf = container_of(src_buf,
 372                                                        struct v4l2_m2m_buffer,
 373                                                        vb);
 374                                 list_add_tail(&m2m_buf->list, buffer_list);
 375                         } else {
 376                                 v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
 377                         }
 378                         continue;
 379                 }
 380
 381                 /* Dump empty buffers */
 382                 if (!vb2_get_plane_payload(&src_buf->vb2_buf, 0)) {
 383                         src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
 384                         v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
 385                         continue;
 386                 }
 387
 388                 /* Buffer start position */
 389                 start = ctx->bitstream_fifo.kfifo.in;
 390
 391                 if (coda_bitstream_try_queue(ctx, src_buf)) {
 392                         /*
 393                          * Source buffer is queued in the bitstream ringbuffer;
 394                          * queue the timestamp and mark source buffer as done
 395                          */
 396                         src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
 397
 398                         meta = kmalloc(sizeof(*meta), GFP_KERNEL);
 399                         if (meta) {
 400                                 meta->sequence = src_buf->sequence;
 401                                 meta->timecode = src_buf->timecode;
 402                                 meta->timestamp = src_buf->vb2_buf.timestamp;
 403                                 meta->start = start;
 404                                 meta->end = ctx->bitstream_fifo.kfifo.in;
 405                                 meta->last = src_buf->flags & V4L2_BUF_FLAG_LAST;
 406                                 if (meta->last)
 407                                         coda_dbg(1, ctx, "marking last meta");
 408                                 spin_lock(&ctx->buffer_meta_lock);
 409                                 list_add_tail(&meta->list,
 410                                               &ctx->buffer_meta_list);
 411                                 ctx->num_metas++;
 412                                 spin_unlock(&ctx->buffer_meta_lock);
 413
 414                                 trace_coda_bit_queue(ctx, src_buf, meta);
 415                         }
 416
 417                         if (buffer_list) {
 418                                 struct v4l2_m2m_buffer *m2m_buf;
 419
 420                                 m2m_buf = container_of(src_buf,
 421                                                        struct v4l2_m2m_buffer,
 422                                                        vb);
 423                                 list_add_tail(&m2m_buf->list, buffer_list);
 424                         } else {
 425                                 v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
 426                         }
 427                 } else {
 428                         break;
 429                 }
 430         }
 431 }
 432
 433 void coda_bit_stream_end_flag(struct coda_ctx *ctx)
 434 {
 435         struct coda_dev *dev = ctx->dev;
 436
 437         ctx->bit_stream_param |= CODA_BIT_STREAM_END_FLAG;
 438
 439         /* If this context is currently running, update the hardware flag */
 440         if ((dev->devtype->product == CODA_960) &&
 441             coda_isbusy(dev) &&
 442             (ctx->idx == coda_read(dev, CODA_REG_BIT_RUN_INDEX))) {
 443                 coda_write(dev, ctx->bit_stream_param,
 444                            CODA_REG_BIT_BIT_STREAM_PARAM);
 445         }
 446 }
 447
 448 static void coda_parabuf_write(struct coda_ctx *ctx, int index, u32 value)
 449 {
 450         struct coda_dev *dev = ctx->dev;
 451         u32 *p = ctx->parabuf.vaddr;
 452
 453         if (dev->devtype->product == CODA_DX6)
 454                 p[index] = value;
 455         else
 456                 p[index ^ 1] = value;
 457 }
 458
 459 static inline int coda_alloc_context_buf(struct coda_ctx *ctx,
 460                                          struct coda_aux_buf *buf, size_t size,
 461                                          const char *name)
 462 {
 463         return coda_alloc_aux_buf(ctx->dev, buf, size, name, ctx->debugfs_entry);
 464 }
 465
 466
 467 static void coda_free_framebuffers(struct coda_ctx *ctx)
 468 {
 469         int i;
 470
 471         for (i = 0; i < CODA_MAX_FRAMEBUFFERS; i++)
 472                 coda_free_aux_buf(ctx->dev, &ctx->internal_frames[i].buf);
 473 }
 474
 475 static int coda_alloc_framebuffers(struct coda_ctx *ctx,
 476                                    struct coda_q_data *q_data, u32 fourcc)
 477 {
 478         struct coda_dev *dev = ctx->dev;
 479         unsigned int ysize, ycbcr_size;
 480         int ret;
 481         int i;
 482
 483         if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 ||
 484             ctx->codec->dst_fourcc == V4L2_PIX_FMT_H264 ||
 485             ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 ||
 486             ctx->codec->dst_fourcc == V4L2_PIX_FMT_MPEG4)
 487                 ysize = round_up(q_data->rect.width, 16) *
 488                         round_up(q_data->rect.height, 16);
 489         else
 490                 ysize = round_up(q_data->rect.width, 8) * q_data->rect.height;
 491
 492         if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP)
 493                 ycbcr_size = round_up(ysize, 4096) + ysize / 2;
 494         else
 495                 ycbcr_size = ysize + ysize / 2;
 496
 497         /* Allocate frame buffers */
 498         for (i = 0; i < ctx->num_internal_frames; i++) {
 499                 size_t size = ycbcr_size;
 500                 char *name;
 501
 502                 /* Add space for mvcol buffers */
 503                 if (dev->devtype->product != CODA_DX6 &&
 504                     (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 ||
 505                      (ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 && i == 0)))
 506                         size += ysize / 4;
 507                 name = kasprintf(GFP_KERNEL, "fb%d", i);
 508                 if (!name) {
 509                         coda_free_framebuffers(ctx);
 510                         return -ENOMEM;
 511                 }
 512                 ret = coda_alloc_context_buf(ctx, &ctx->internal_frames[i].buf,
 513                                              size, name);
 514                 kfree(name);
 515                 if (ret < 0) {
 516                         coda_free_framebuffers(ctx);
 517                         return ret;
 518                 }
 519         }
 520
 521         /* Register frame buffers in the parameter buffer */
 522         for (i = 0; i < ctx->num_internal_frames; i++) {
 523                 u32 y, cb, cr, mvcol;
 524
 525                 /* Start addresses of Y, Cb, Cr planes */
 526                 y = ctx->internal_frames[i].buf.paddr;
 527                 cb = y + ysize;
 528                 cr = y + ysize + ysize/4;
 529                 mvcol = y + ysize + ysize/4 + ysize/4;
 530                 if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP) {
 531                         cb = round_up(cb, 4096);
 532                         mvcol = cb + ysize/2;
 533                         cr = 0;
 534                         /* Packed 20-bit MSB of base addresses */
 535                         /* YYYYYCCC, CCyyyyyc, cccc.... */
 536                         y = (y & 0xfffff000) | cb >> 20;
 537                         cb = (cb & 0x000ff000) << 12;
 538                 }
 539                 coda_parabuf_write(ctx, i * 3 + 0, y);
 540                 coda_parabuf_write(ctx, i * 3 + 1, cb);
 541                 coda_parabuf_write(ctx, i * 3 + 2, cr);
 542
 543                 if (dev->devtype->product == CODA_DX6)
 544                         continue;
 545
 546                 /* mvcol buffer for h.264 and mpeg4 */
 547                 if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264)
 548                         coda_parabuf_write(ctx, 96 + i, mvcol);
 549                 if (ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 && i == 0)
 550                         coda_parabuf_write(ctx, 97, mvcol);
 551         }
 552
 553         return 0;
 554 }
 555
 556 static void coda_free_context_buffers(struct coda_ctx *ctx)
 557 {
 558         struct coda_dev *dev = ctx->dev;
 559
 560         coda_free_aux_buf(dev, &ctx->slicebuf);
 561         coda_free_aux_buf(dev, &ctx->psbuf);
 562         if (dev->devtype->product != CODA_DX6)
 563                 coda_free_aux_buf(dev, &ctx->workbuf);
 564         coda_free_aux_buf(dev, &ctx->parabuf);
 565 }
 566
 567 static int coda_alloc_context_buffers(struct coda_ctx *ctx,
 568                                       struct coda_q_data *q_data)
 569 {
 570         struct coda_dev *dev = ctx->dev;
 571         size_t size;
 572         int ret;
 573
 574         if (!ctx->parabuf.vaddr) {
 575                 ret = coda_alloc_context_buf(ctx, &ctx->parabuf,
 576                                              CODA_PARA_BUF_SIZE, "parabuf");
 577                 if (ret < 0)
 578                         return ret;
 579         }
 580
 581         if (dev->devtype->product == CODA_DX6)
 582                 return 0;
 583
 584         if (!ctx->slicebuf.vaddr && q_data->fourcc == V4L2_PIX_FMT_H264) {
 585                 /* worst case slice size */
 586                 size = (DIV_ROUND_UP(q_data->rect.width, 16) *
 587                         DIV_ROUND_UP(q_data->rect.height, 16)) * 3200 / 8 + 512;
 588                 ret = coda_alloc_context_buf(ctx, &ctx->slicebuf, size,
 589                                              "slicebuf");
 590                 if (ret < 0)
 591                         goto err;
 592         }
 593
 594         if (!ctx->psbuf.vaddr && (dev->devtype->product == CODA_HX4 ||
 595                                   dev->devtype->product == CODA_7541)) {
 596                 ret = coda_alloc_context_buf(ctx, &ctx->psbuf,
 597                                              CODA7_PS_BUF_SIZE, "psbuf");
 598                 if (ret < 0)
 599                         goto err;
 600         }
 601
 602         if (!ctx->workbuf.vaddr) {
 603                 size = dev->devtype->workbuf_size;
 604                 if (dev->devtype->product == CODA_960 &&
 605                     q_data->fourcc == V4L2_PIX_FMT_H264)
 606                         size += CODA9_PS_SAVE_SIZE;
 607                 ret = coda_alloc_context_buf(ctx, &ctx->workbuf, size,
 608                                              "workbuf");
 609                 if (ret < 0)
 610                         goto err;
 611         }
 612
 613         return 0;
 614
 615 err:
 616         coda_free_context_buffers(ctx);
 617         return ret;
 618 }
 619
 620 static int coda_encode_header(struct coda_ctx *ctx, struct vb2_v4l2_buffer *buf,
 621                               int header_code, u8 *header, int *size)
 622 {
 623         struct vb2_buffer *vb = &buf->vb2_buf;
 624         struct coda_dev *dev = ctx->dev;
 625         struct coda_q_data *q_data_src;
 626         struct v4l2_rect *r;
 627         size_t bufsize;
 628         int ret;
 629         int i;
 630
 631         if (dev->devtype->product == CODA_960)
 632                 memset(vb2_plane_vaddr(vb, 0), 0, 64);
 633
 634         coda_write(dev, vb2_dma_contig_plane_dma_addr(vb, 0),
 635                    CODA_CMD_ENC_HEADER_BB_START);
 636         bufsize = vb2_plane_size(vb, 0);
 637         if (dev->devtype->product == CODA_960)
 638                 bufsize /= 1024;
 639         coda_write(dev, bufsize, CODA_CMD_ENC_HEADER_BB_SIZE);
 640         if (dev->devtype->product == CODA_960 &&
 641             ctx->codec->dst_fourcc == V4L2_PIX_FMT_H264 &&
 642             header_code == CODA_HEADER_H264_SPS) {
 643                 q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 644                 r = &q_data_src->rect;
 645
 646                 if (r->width % 16 || r->height % 16) {
 647                         u32 crop_right = round_up(r->width, 16) -  r->width;
 648                         u32 crop_bottom = round_up(r->height, 16) - r->height;
 649
 650                         coda_write(dev, crop_right,
 651                                    CODA9_CMD_ENC_HEADER_FRAME_CROP_H);
 652                         coda_write(dev, crop_bottom,
 653                                    CODA9_CMD_ENC_HEADER_FRAME_CROP_V);
 654                         header_code |= CODA9_HEADER_FRAME_CROP;
 655                 }
 656         }
 657         coda_write(dev, header_code, CODA_CMD_ENC_HEADER_CODE);
 658         ret = coda_command_sync(ctx, CODA_COMMAND_ENCODE_HEADER);
 659         if (ret < 0) {
 660                 v4l2_err(&dev->v4l2_dev, "CODA_COMMAND_ENCODE_HEADER timeout\n");
 661                 return ret;
 662         }
 663
 664         if (dev->devtype->product == CODA_960) {
 665                 for (i = 63; i > 0; i--)
 666                         if (((char *)vb2_plane_vaddr(vb, 0))[i] != 0)
 667                                 break;
 668                 *size = i + 1;
 669         } else {
 670                 *size = coda_read(dev, CODA_REG_BIT_WR_PTR(ctx->reg_idx)) -
 671                         coda_read(dev, CODA_CMD_ENC_HEADER_BB_START);
 672         }
 673         memcpy(header, vb2_plane_vaddr(vb, 0), *size);
 674
 675         return 0;
 676 }
 677
 678 static u32 coda_slice_mode(struct coda_ctx *ctx)
 679 {
 680         int size, unit;
 681
 682         switch (ctx->params.slice_mode) {
 683         case V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_SINGLE:
 684         default:
 685                 return 0;
 686         case V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_MAX_MB:
 687                 size = ctx->params.slice_max_mb;
 688                 unit = 1;
 689                 break;
 690         case V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_MAX_BYTES:
 691                 size = ctx->params.slice_max_bits;
 692                 unit = 0;
 693                 break;
 694         }
 695
 696         return ((size & CODA_SLICING_SIZE_MASK) << CODA_SLICING_SIZE_OFFSET) |
 697                ((unit & CODA_SLICING_UNIT_MASK) << CODA_SLICING_UNIT_OFFSET) |
 698                ((1 & CODA_SLICING_MODE_MASK) << CODA_SLICING_MODE_OFFSET);
 699 }
 700
 701 static int coda_enc_param_change(struct coda_ctx *ctx)
 702 {
 703         struct coda_dev *dev = ctx->dev;
 704         u32 change_enable = 0;
 705         u32 success;
 706         int ret;
 707
 708         if (ctx->params.gop_size_changed) {
 709                 change_enable |= CODA_PARAM_CHANGE_RC_GOP;
 710                 coda_write(dev, ctx->params.gop_size,
 711                            CODA_CMD_ENC_PARAM_RC_GOP);
 712                 ctx->gopcounter = ctx->params.gop_size - 1;
 713                 ctx->params.gop_size_changed = false;
 714         }
 715         if (ctx->params.h264_intra_qp_changed) {
 716                 coda_dbg(1, ctx, "parameter change: intra Qp %u\n",
 717                          ctx->params.h264_intra_qp);
 718
 719                 if (ctx->params.bitrate) {
 720                         change_enable |= CODA_PARAM_CHANGE_RC_INTRA_QP;
 721                         coda_write(dev, ctx->params.h264_intra_qp,
 722                                    CODA_CMD_ENC_PARAM_RC_INTRA_QP);
 723                 }
 724                 ctx->params.h264_intra_qp_changed = false;
 725         }
 726         if (ctx->params.bitrate_changed) {
 727                 coda_dbg(1, ctx, "parameter change: bitrate %u kbit/s\n",
 728                          ctx->params.bitrate);
 729                 change_enable |= CODA_PARAM_CHANGE_RC_BITRATE;
 730                 coda_write(dev, ctx->params.bitrate,
 731                            CODA_CMD_ENC_PARAM_RC_BITRATE);
 732                 ctx->params.bitrate_changed = false;
 733         }
 734         if (ctx->params.framerate_changed) {
 735                 coda_dbg(1, ctx, "parameter change: frame rate %u/%u Hz\n",
 736                          ctx->params.framerate & 0xffff,
 737                          (ctx->params.framerate >> 16) + 1);
 738                 change_enable |= CODA_PARAM_CHANGE_RC_FRAME_RATE;
 739                 coda_write(dev, ctx->params.framerate,
 740                            CODA_CMD_ENC_PARAM_RC_FRAME_RATE);
 741                 ctx->params.framerate_changed = false;
 742         }
 743         if (ctx->params.intra_refresh_changed) {
 744                 coda_dbg(1, ctx, "parameter change: intra refresh MBs %u\n",
 745                          ctx->params.intra_refresh);
 746                 change_enable |= CODA_PARAM_CHANGE_INTRA_MB_NUM;
 747                 coda_write(dev, ctx->params.intra_refresh,
 748                            CODA_CMD_ENC_PARAM_INTRA_MB_NUM);
 749                 ctx->params.intra_refresh_changed = false;
 750         }
 751         if (ctx->params.slice_mode_changed) {
 752                 change_enable |= CODA_PARAM_CHANGE_SLICE_MODE;
 753                 coda_write(dev, coda_slice_mode(ctx),
 754                            CODA_CMD_ENC_PARAM_SLICE_MODE);
 755                 ctx->params.slice_mode_changed = false;
 756         }
 757
 758         if (!change_enable)
 759                 return 0;
 760
 761         coda_write(dev, change_enable, CODA_CMD_ENC_PARAM_CHANGE_ENABLE);
 762
 763         ret = coda_command_sync(ctx, CODA_COMMAND_RC_CHANGE_PARAMETER);
 764         if (ret < 0)
 765                 return ret;
 766
 767         success = coda_read(dev, CODA_RET_ENC_PARAM_CHANGE_SUCCESS);
 768         if (success != 1)
 769                 coda_dbg(1, ctx, "parameter change failed: %u\n", success);
 770
 771         return 0;
 772 }
 773
 774 static phys_addr_t coda_iram_alloc(struct coda_iram_info *iram, size_t size)
 775 {
 776         phys_addr_t ret;
 777
 778         size = round_up(size, 1024);
 779         if (size > iram->remaining)
 780                 return 0;
 781         iram->remaining -= size;
 782
 783         ret = iram->next_paddr;
 784         iram->next_paddr += size;
 785
 786         return ret;
 787 }
 788
 789 static void coda_setup_iram(struct coda_ctx *ctx)
 790 {
 791         struct coda_iram_info *iram_info = &ctx->iram_info;
 792         struct coda_dev *dev = ctx->dev;
 793         int w64, w128;
 794         int mb_width;
 795         int dbk_bits;
 796         int bit_bits;
 797         int ip_bits;
 798         int me_bits;
 799
 800         memset(iram_info, 0, sizeof(*iram_info));
 801         iram_info->next_paddr = dev->iram.paddr;
 802         iram_info->remaining = dev->iram.size;
 803
 804         if (!dev->iram.vaddr)
 805                 return;
 806
 807         switch (dev->devtype->product) {
 808         case CODA_HX4:
 809                 dbk_bits = CODA7_USE_HOST_DBK_ENABLE;
 810                 bit_bits = CODA7_USE_HOST_BIT_ENABLE;
 811                 ip_bits = CODA7_USE_HOST_IP_ENABLE;
 812                 me_bits = CODA7_USE_HOST_ME_ENABLE;
 813                 break;
 814         case CODA_7541:
 815                 dbk_bits = CODA7_USE_HOST_DBK_ENABLE | CODA7_USE_DBK_ENABLE;
 816                 bit_bits = CODA7_USE_HOST_BIT_ENABLE | CODA7_USE_BIT_ENABLE;
 817                 ip_bits = CODA7_USE_HOST_IP_ENABLE | CODA7_USE_IP_ENABLE;
 818                 me_bits = CODA7_USE_HOST_ME_ENABLE | CODA7_USE_ME_ENABLE;
 819                 break;
 820         case CODA_960:
 821                 dbk_bits = CODA9_USE_HOST_DBK_ENABLE | CODA9_USE_DBK_ENABLE;
 822                 bit_bits = CODA9_USE_HOST_BIT_ENABLE | CODA7_USE_BIT_ENABLE;
 823                 ip_bits = CODA9_USE_HOST_IP_ENABLE | CODA7_USE_IP_ENABLE;
 824                 me_bits = 0;
 825                 break;
 826         default: /* CODA_DX6 */
 827                 return;
 828         }
 829
 830         if (ctx->inst_type == CODA_INST_ENCODER) {
 831                 struct coda_q_data *q_data_src;
 832
 833                 q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 834                 mb_width = DIV_ROUND_UP(q_data_src->rect.width, 16);
 835                 w128 = mb_width * 128;
 836                 w64 = mb_width * 64;
 837
 838                 /* Prioritize in case IRAM is too small for everything */
 839                 if (dev->devtype->product == CODA_HX4 ||
 840                     dev->devtype->product == CODA_7541) {
 841                         iram_info->search_ram_size = round_up(mb_width * 16 *
 842                                                               36 + 2048, 1024);
 843                         iram_info->search_ram_paddr = coda_iram_alloc(iram_info,
 844                                                 iram_info->search_ram_size);
 845                         if (!iram_info->search_ram_paddr) {
 846                                 pr_err("IRAM is smaller than the search ram size\n");
 847                                 goto out;
 848                         }
 849                         iram_info->axi_sram_use |= me_bits;
 850                 }
 851
 852                 /* Only H.264BP and H.263P3 are considered */
 853                 iram_info->buf_dbk_y_use = coda_iram_alloc(iram_info, w64);
 854                 iram_info->buf_dbk_c_use = coda_iram_alloc(iram_info, w64);
 855                 if (!iram_info->buf_dbk_y_use || !iram_info->buf_dbk_c_use)
 856                         goto out;
 857                 iram_info->axi_sram_use |= dbk_bits;
 858
 859                 iram_info->buf_bit_use = coda_iram_alloc(iram_info, w128);
 860                 if (!iram_info->buf_bit_use)
 861                         goto out;
 862                 iram_info->axi_sram_use |= bit_bits;
 863
 864                 iram_info->buf_ip_ac_dc_use = coda_iram_alloc(iram_info, w128);
 865                 if (!iram_info->buf_ip_ac_dc_use)
 866                         goto out;
 867                 iram_info->axi_sram_use |= ip_bits;
 868
 869                 /* OVL and BTP disabled for encoder */
 870         } else if (ctx->inst_type == CODA_INST_DECODER) {
 871                 struct coda_q_data *q_data_dst;
 872
 873                 q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 874                 mb_width = DIV_ROUND_UP(q_data_dst->width, 16);
 875                 w128 = mb_width * 128;
 876
 877                 iram_info->buf_dbk_y_use = coda_iram_alloc(iram_info, w128);
 878                 iram_info->buf_dbk_c_use = coda_iram_alloc(iram_info, w128);
 879                 if (!iram_info->buf_dbk_y_use || !iram_info->buf_dbk_c_use)
 880                         goto out;
 881                 iram_info->axi_sram_use |= dbk_bits;
 882
 883                 iram_info->buf_bit_use = coda_iram_alloc(iram_info, w128);
 884                 if (!iram_info->buf_bit_use)
 885                         goto out;
 886                 iram_info->axi_sram_use |= bit_bits;
 887
 888                 iram_info->buf_ip_ac_dc_use = coda_iram_alloc(iram_info, w128);
 889                 if (!iram_info->buf_ip_ac_dc_use)
 890                         goto out;
 891                 iram_info->axi_sram_use |= ip_bits;
 892
 893                 /* OVL and BTP unused as there is no VC1 support yet */
 894         }
 895
 896 out:
 897         if (!(iram_info->axi_sram_use & CODA7_USE_HOST_IP_ENABLE))
 898                 coda_dbg(1, ctx, "IRAM smaller than needed\n");
 899
 900         if (dev->devtype->product == CODA_HX4 ||
 901             dev->devtype->product == CODA_7541) {
 902                 /* TODO - Enabling these causes picture errors on CODA7541 */
 903                 if (ctx->inst_type == CODA_INST_DECODER) {
 904                         /* fw 1.4.50 */
 905                         iram_info->axi_sram_use &= ~(CODA7_USE_HOST_IP_ENABLE |
 906                                                      CODA7_USE_IP_ENABLE);
 907                 } else {
 908                         /* fw 13.4.29 */
 909                         iram_info->axi_sram_use &= ~(CODA7_USE_HOST_IP_ENABLE |
 910                                                      CODA7_USE_HOST_DBK_ENABLE |
 911                                                      CODA7_USE_IP_ENABLE |
 912                                                      CODA7_USE_DBK_ENABLE);
 913                 }
 914         }
 915 }
 916
 917 static u32 coda_supported_firmwares[] = {
 918         CODA_FIRMWARE_VERNUM(CODA_DX6, 2, 2, 5),
 919         CODA_FIRMWARE_VERNUM(CODA_HX4, 1, 4, 50),
 920         CODA_FIRMWARE_VERNUM(CODA_7541, 1, 4, 50),
 921         CODA_FIRMWARE_VERNUM(CODA_960, 2, 1, 5),
 922         CODA_FIRMWARE_VERNUM(CODA_960, 2, 1, 9),
 923         CODA_FIRMWARE_VERNUM(CODA_960, 2, 3, 10),
 924         CODA_FIRMWARE_VERNUM(CODA_960, 3, 1, 1),
 925 };
 926
 927 static bool coda_firmware_supported(u32 vernum)
 928 {
 929         int i;
 930
 931         for (i = 0; i < ARRAY_SIZE(coda_supported_firmwares); i++)
 932                 if (vernum == coda_supported_firmwares[i])
 933                         return true;
 934         return false;
 935 }
 936
 937 int coda_check_firmware(struct coda_dev *dev)
 938 {
 939         u16 product, major, minor, release;
 940         u32 data;
 941         int ret;
 942
 943         ret = clk_prepare_enable(dev->clk_per);
 944         if (ret)
 945                 goto err_clk_per;
 946
 947         ret = clk_prepare_enable(dev->clk_ahb);
 948         if (ret)
 949                 goto err_clk_ahb;
 950
 951         coda_write(dev, 0, CODA_CMD_FIRMWARE_VERNUM);
 952         coda_write(dev, CODA_REG_BIT_BUSY_FLAG, CODA_REG_BIT_BUSY);
 953         coda_write(dev, 0, CODA_REG_BIT_RUN_INDEX);
 954         coda_write(dev, 0, CODA_REG_BIT_RUN_COD_STD);
 955         coda_write(dev, CODA_COMMAND_FIRMWARE_GET, CODA_REG_BIT_RUN_COMMAND);
 956         if (coda_wait_timeout(dev)) {
 957                 v4l2_err(&dev->v4l2_dev, "firmware get command error\n");
 958                 ret = -EIO;
 959                 goto err_run_cmd;
 960         }
 961
 962         if (dev->devtype->product == CODA_960) {
 963                 data = coda_read(dev, CODA9_CMD_FIRMWARE_CODE_REV);
 964                 v4l2_info(&dev->v4l2_dev, "Firmware code revision: %d\n",
 965                           data);
 966         }
 967
 968         /* Check we are compatible with the loaded firmware */
 969         data = coda_read(dev, CODA_CMD_FIRMWARE_VERNUM);
 970         product = CODA_FIRMWARE_PRODUCT(data);
 971         major = CODA_FIRMWARE_MAJOR(data);
 972         minor = CODA_FIRMWARE_MINOR(data);
 973         release = CODA_FIRMWARE_RELEASE(data);
 974
 975         clk_disable_unprepare(dev->clk_per);
 976         clk_disable_unprepare(dev->clk_ahb);
 977
 978         if (product != dev->devtype->product) {
 979                 v4l2_err(&dev->v4l2_dev,
 980                          "Wrong firmware. Hw: %s, Fw: %s, Version: %u.%u.%u\n",
 981                          coda_product_name(dev->devtype->product),
 982                          coda_product_name(product), major, minor, release);
 983                 return -EINVAL;
 984         }
 985
 986         v4l2_info(&dev->v4l2_dev, "Initialized %s.\n",
 987                   coda_product_name(product));
 988
 989         if (coda_firmware_supported(data)) {
 990                 v4l2_info(&dev->v4l2_dev, "Firmware version: %u.%u.%u\n",
 991                           major, minor, release);
 992         } else {
 993                 v4l2_warn(&dev->v4l2_dev,
 994                           "Unsupported firmware version: %u.%u.%u\n",
 995                           major, minor, release);
 996         }
 997
 998         return 0;
 999
1000 err_run_cmd:
1001         clk_disable_unprepare(dev->clk_ahb);
1002 err_clk_ahb:
1003         clk_disable_unprepare(dev->clk_per);
1004 err_clk_per:
1005         return ret;
1006 }
1007
1008 static void coda9_set_frame_cache(struct coda_ctx *ctx, u32 fourcc)
1009 {
1010         u32 cache_size, cache_config;
1011
1012         if (ctx->tiled_map_type == GDI_LINEAR_FRAME_MAP) {
1013                 /* Luma 2x0 page, 2x6 cache, chroma 2x0 page, 2x4 cache size */
1014                 cache_size = 0x20262024;
1015                 cache_config = 2 << CODA9_CACHE_PAGEMERGE_OFFSET;
1016         } else {
1017                 /* Luma 0x2 page, 4x4 cache, chroma 0x2 page, 4x3 cache size */
1018                 cache_size = 0x02440243;
1019                 cache_config = 1 << CODA9_CACHE_PAGEMERGE_OFFSET;
1020         }
1021         coda_write(ctx->dev, cache_size, CODA9_CMD_SET_FRAME_CACHE_SIZE);
1022         if (fourcc == V4L2_PIX_FMT_NV12 || fourcc == V4L2_PIX_FMT_YUYV) {
1023                 cache_config |= 32 << CODA9_CACHE_LUMA_BUFFER_SIZE_OFFSET |
1024                                 16 << CODA9_CACHE_CR_BUFFER_SIZE_OFFSET |
1025                                 0 << CODA9_CACHE_CB_BUFFER_SIZE_OFFSET;
1026         } else {
1027                 cache_config |= 32 << CODA9_CACHE_LUMA_BUFFER_SIZE_OFFSET |
1028                                 8 << CODA9_CACHE_CR_BUFFER_SIZE_OFFSET |
1029                                 8 << CODA9_CACHE_CB_BUFFER_SIZE_OFFSET;
1030         }
1031         coda_write(ctx->dev, cache_config, CODA9_CMD_SET_FRAME_CACHE_CONFIG);
1032 }
1033
1034 /*
1035  * Encoder context operations
1036  */
1037
1038 static int coda_encoder_reqbufs(struct coda_ctx *ctx,
1039                                 struct v4l2_requestbuffers *rb)
1040 {
1041         struct coda_q_data *q_data_src;
1042         int ret;
1043
1044         if (rb->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
1045                 return 0;
1046
1047         if (rb->count) {
1048                 q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
1049                 ret = coda_alloc_context_buffers(ctx, q_data_src);
1050                 if (ret < 0)
1051                         return ret;
1052         } else {
1053                 coda_free_context_buffers(ctx);
1054         }
1055
1056         return 0;
1057 }
1058
1059 static int coda_start_encoding(struct coda_ctx *ctx)
1060 {
1061         struct coda_dev *dev = ctx->dev;
1062         struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1063         struct coda_q_data *q_data_src, *q_data_dst;
1064         u32 bitstream_buf, bitstream_size;
1065         struct vb2_v4l2_buffer *buf;
1066         int gamma, ret, value;
1067         u32 dst_fourcc;
1068         int num_fb;
1069         u32 stride;
1070
1071         q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
1072         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
1073         dst_fourcc = q_data_dst->fourcc;
1074
1075         buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
1076         bitstream_buf = vb2_dma_contig_plane_dma_addr(&buf->vb2_buf, 0);
1077         bitstream_size = q_data_dst->sizeimage;
1078
1079         if (!coda_is_initialized(dev)) {
1080                 v4l2_err(v4l2_dev, "coda is not initialized.\n");
1081                 return -EFAULT;
1082         }
1083
1084         if (dst_fourcc == V4L2_PIX_FMT_JPEG) {
1085                 if (!ctx->params.jpeg_qmat_tab[0]) {
1086                         ctx->params.jpeg_qmat_tab[0] = kmalloc(64, GFP_KERNEL);
1087                         if (!ctx->params.jpeg_qmat_tab[0])
1088                                 return -ENOMEM;
1089                 }
1090                 if (!ctx->params.jpeg_qmat_tab[1]) {
1091                         ctx->params.jpeg_qmat_tab[1] = kmalloc(64, GFP_KERNEL);
1092                         if (!ctx->params.jpeg_qmat_tab[1])
1093                                 return -ENOMEM;
1094                 }
1095                 coda_set_jpeg_compression_quality(ctx, ctx->params.jpeg_quality);
1096         }
1097
1098         mutex_lock(&dev->coda_mutex);
1099
1100         coda_write(dev, ctx->parabuf.paddr, CODA_REG_BIT_PARA_BUF_ADDR);
1101         coda_write(dev, bitstream_buf, CODA_REG_BIT_RD_PTR(ctx->reg_idx));
1102         coda_write(dev, bitstream_buf, CODA_REG_BIT_WR_PTR(ctx->reg_idx));
1103         switch (dev->devtype->product) {
1104         case CODA_DX6:
1105                 coda_write(dev, CODADX6_STREAM_BUF_DYNALLOC_EN |
1106                         CODADX6_STREAM_BUF_PIC_RESET, CODA_REG_BIT_STREAM_CTRL);
1107                 break;
1108         case CODA_960:
1109                 coda_write(dev, 0, CODA9_GDI_WPROT_RGN_EN);
1110                 fallthrough;
1111         case CODA_HX4:
1112         case CODA_7541:
1113                 coda_write(dev, CODA7_STREAM_BUF_DYNALLOC_EN |
1114                         CODA7_STREAM_BUF_PIC_RESET, CODA_REG_BIT_STREAM_CTRL);
1115                 break;
1116         }
1117
1118         ctx->frame_mem_ctrl &= ~(CODA_FRAME_CHROMA_INTERLEAVE | (0x3 << 9) |
1119                                  CODA9_FRAME_TILED2LINEAR);
1120         if (q_data_src->fourcc == V4L2_PIX_FMT_NV12)
1121                 ctx->frame_mem_ctrl |= CODA_FRAME_CHROMA_INTERLEAVE;
1122         if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP)
1123                 ctx->frame_mem_ctrl |= (0x3 << 9) | CODA9_FRAME_TILED2LINEAR;
1124         coda_write(dev, ctx->frame_mem_ctrl, CODA_REG_BIT_FRAME_MEM_CTRL);
1125
1126         if (dev->devtype->product == CODA_DX6) {
1127                 /* Configure the coda */
1128                 coda_write(dev, dev->iram.paddr,
1129                            CODADX6_REG_BIT_SEARCH_RAM_BASE_ADDR);
1130         }
1131
1132         /* Could set rotation here if needed */
1133         value = 0;
1134         switch (dev->devtype->product) {
1135         case CODA_DX6:
1136                 value = (q_data_src->rect.width & CODADX6_PICWIDTH_MASK)
1137                         << CODADX6_PICWIDTH_OFFSET;
1138                 value |= (q_data_src->rect.height & CODADX6_PICHEIGHT_MASK)
1139                          << CODA_PICHEIGHT_OFFSET;
1140                 break;
1141         case CODA_HX4:
1142         case CODA_7541:
1143                 if (dst_fourcc == V4L2_PIX_FMT_H264) {
1144                         value = (round_up(q_data_src->rect.width, 16) &
1145                                  CODA7_PICWIDTH_MASK) << CODA7_PICWIDTH_OFFSET;
1146                         value |= (round_up(q_data_src->rect.height, 16) &
1147                                  CODA7_PICHEIGHT_MASK) << CODA_PICHEIGHT_OFFSET;
1148                         break;
1149                 }
1150                 fallthrough;
1151         case CODA_960:
1152                 value = (q_data_src->rect.width & CODA7_PICWIDTH_MASK)
1153                         << CODA7_PICWIDTH_OFFSET;
1154                 value |= (q_data_src->rect.height & CODA7_PICHEIGHT_MASK)
1155                          << CODA_PICHEIGHT_OFFSET;
1156         }
1157         coda_write(dev, value, CODA_CMD_ENC_SEQ_SRC_SIZE);
1158         if (dst_fourcc == V4L2_PIX_FMT_JPEG)
1159                 ctx->params.framerate = 0;
1160         coda_write(dev, ctx->params.framerate,
1161                    CODA_CMD_ENC_SEQ_SRC_F_RATE);
1162
1163         ctx->params.codec_mode = ctx->codec->mode;
1164         switch (dst_fourcc) {
1165         case V4L2_PIX_FMT_MPEG4:
1166                 if (dev->devtype->product == CODA_960)
1167                         coda_write(dev, CODA9_STD_MPEG4,
1168                                    CODA_CMD_ENC_SEQ_COD_STD);
1169                 else
1170                         coda_write(dev, CODA_STD_MPEG4,
1171                                    CODA_CMD_ENC_SEQ_COD_STD);
1172                 coda_write(dev, 0, CODA_CMD_ENC_SEQ_MP4_PARA);
1173                 break;
1174         case V4L2_PIX_FMT_H264:
1175                 if (dev->devtype->product == CODA_960)
1176                         coda_write(dev, CODA9_STD_H264,
1177                                    CODA_CMD_ENC_SEQ_COD_STD);
1178                 else
1179                         coda_write(dev, CODA_STD_H264,
1180                                    CODA_CMD_ENC_SEQ_COD_STD);
1181                 value = ((ctx->params.h264_disable_deblocking_filter_idc &
1182                           CODA_264PARAM_DISABLEDEBLK_MASK) <<
1183                          CODA_264PARAM_DISABLEDEBLK_OFFSET) |
1184                         ((ctx->params.h264_slice_alpha_c0_offset_div2 &
1185                           CODA_264PARAM_DEBLKFILTEROFFSETALPHA_MASK) <<
1186                          CODA_264PARAM_DEBLKFILTEROFFSETALPHA_OFFSET) |
1187                         ((ctx->params.h264_slice_beta_offset_div2 &
1188                           CODA_264PARAM_DEBLKFILTEROFFSETBETA_MASK) <<
1189                          CODA_264PARAM_DEBLKFILTEROFFSETBETA_OFFSET) |
1190                         (ctx->params.h264_constrained_intra_pred_flag <<
1191                          CODA_264PARAM_CONSTRAINEDINTRAPREDFLAG_OFFSET) |
1192                         (ctx->params.h264_chroma_qp_index_offset &
1193                          CODA_264PARAM_CHROMAQPOFFSET_MASK);
1194                 coda_write(dev, value, CODA_CMD_ENC_SEQ_264_PARA);
1195                 break;
1196         case V4L2_PIX_FMT_JPEG:
1197                 coda_write(dev, 0, CODA_CMD_ENC_SEQ_JPG_PARA);
1198                 coda_write(dev, ctx->params.jpeg_restart_interval,
1199                                 CODA_CMD_ENC_SEQ_JPG_RST_INTERVAL);
1200                 coda_write(dev, 0, CODA_CMD_ENC_SEQ_JPG_THUMB_EN);
1201                 coda_write(dev, 0, CODA_CMD_ENC_SEQ_JPG_THUMB_SIZE);
1202                 coda_write(dev, 0, CODA_CMD_ENC_SEQ_JPG_THUMB_OFFSET);
1203
1204                 coda_jpeg_write_tables(ctx);
1205                 break;
1206         default:
1207                 v4l2_err(v4l2_dev,
1208                          "dst format (0x%08x) invalid.\n", dst_fourcc);
1209                 ret = -EINVAL;
1210                 goto out;
1211         }
1212
1213         /*
1214          * slice mode and GOP size registers are used for thumb size/offset
1215          * in JPEG mode
1216          */
1217         if (dst_fourcc != V4L2_PIX_FMT_JPEG) {
1218                 value = coda_slice_mode(ctx);
1219                 coda_write(dev, value, CODA_CMD_ENC_SEQ_SLICE_MODE);
1220                 value = ctx->params.gop_size;
1221                 coda_write(dev, value, CODA_CMD_ENC_SEQ_GOP_SIZE);
1222         }
1223
1224         if (ctx->params.bitrate && (ctx->params.frame_rc_enable ||
1225                                     ctx->params.mb_rc_enable)) {
1226                 ctx->params.bitrate_changed = false;
1227                 ctx->params.h264_intra_qp_changed = false;
1228
1229                 /* Rate control enabled */
1230                 value = (ctx->params.bitrate & CODA_RATECONTROL_BITRATE_MASK)
1231                         << CODA_RATECONTROL_BITRATE_OFFSET;
1232                 value |=  1 & CODA_RATECONTROL_ENABLE_MASK;
1233                 value |= (ctx->params.vbv_delay &
1234                           CODA_RATECONTROL_INITIALDELAY_MASK)
1235                          << CODA_RATECONTROL_INITIALDELAY_OFFSET;
1236                 if (dev->devtype->product == CODA_960)
1237                         value |= BIT(31); /* disable autoskip */
1238         } else {
1239                 value = 0;
1240         }
1241         coda_write(dev, value, CODA_CMD_ENC_SEQ_RC_PARA);
1242
1243         coda_write(dev, ctx->params.vbv_size, CODA_CMD_ENC_SEQ_RC_BUF_SIZE);
1244         coda_write(dev, ctx->params.intra_refresh,
1245                    CODA_CMD_ENC_SEQ_INTRA_REFRESH);
1246
1247         coda_write(dev, bitstream_buf, CODA_CMD_ENC_SEQ_BB_START);
1248         coda_write(dev, bitstream_size / 1024, CODA_CMD_ENC_SEQ_BB_SIZE);
1249
1250
1251         value = 0;
1252         if (dev->devtype->product == CODA_960)
1253                 gamma = CODA9_DEFAULT_GAMMA;
1254         else
1255                 gamma = CODA_DEFAULT_GAMMA;
1256         if (gamma > 0) {
1257                 coda_write(dev, (gamma & CODA_GAMMA_MASK) << CODA_GAMMA_OFFSET,
1258                            CODA_CMD_ENC_SEQ_RC_GAMMA);
1259         }
1260
1261         if (ctx->params.h264_min_qp || ctx->params.h264_max_qp) {
1262                 coda_write(dev,
1263                            ctx->params.h264_min_qp << CODA_QPMIN_OFFSET |
1264                            ctx->params.h264_max_qp << CODA_QPMAX_OFFSET,
1265                            CODA_CMD_ENC_SEQ_RC_QP_MIN_MAX);
1266         }
1267         if (dev->devtype->product == CODA_960) {
1268                 if (ctx->params.h264_max_qp)
1269                         value |= 1 << CODA9_OPTION_RCQPMAX_OFFSET;
1270                 if (CODA_DEFAULT_GAMMA > 0)
1271                         value |= 1 << CODA9_OPTION_GAMMA_OFFSET;
1272         } else {
1273                 if (CODA_DEFAULT_GAMMA > 0) {
1274                         if (dev->devtype->product == CODA_DX6)
1275                                 value |= 1 << CODADX6_OPTION_GAMMA_OFFSET;
1276                         else
1277                                 value |= 1 << CODA7_OPTION_GAMMA_OFFSET;
1278                 }
1279                 if (ctx->params.h264_min_qp)
1280                         value |= 1 << CODA7_OPTION_RCQPMIN_OFFSET;
1281                 if (ctx->params.h264_max_qp)
1282                         value |= 1 << CODA7_OPTION_RCQPMAX_OFFSET;
1283         }
1284         coda_write(dev, value, CODA_CMD_ENC_SEQ_OPTION);
1285
1286         if (ctx->params.frame_rc_enable && !ctx->params.mb_rc_enable)
1287                 value = 1;
1288         else
1289                 value = 0;
1290         coda_write(dev, value, CODA_CMD_ENC_SEQ_RC_INTERVAL_MODE);
1291
1292         coda_setup_iram(ctx);
1293
1294         if (dst_fourcc == V4L2_PIX_FMT_H264) {
1295                 switch (dev->devtype->product) {
1296                 case CODA_DX6:
1297                         value = FMO_SLICE_SAVE_BUF_SIZE << 7;
1298                         coda_write(dev, value, CODADX6_CMD_ENC_SEQ_FMO);
1299                         break;
1300                 case CODA_HX4:
1301                 case CODA_7541:
1302                         coda_write(dev, ctx->iram_info.search_ram_paddr,
1303                                         CODA7_CMD_ENC_SEQ_SEARCH_BASE);
1304                         coda_write(dev, ctx->iram_info.search_ram_size,
1305                                         CODA7_CMD_ENC_SEQ_SEARCH_SIZE);
1306                         break;
1307                 case CODA_960:
1308                         coda_write(dev, 0, CODA9_CMD_ENC_SEQ_ME_OPTION);
1309                         coda_write(dev, 0, CODA9_CMD_ENC_SEQ_INTRA_WEIGHT);
1310                 }
1311         }
1312
1313         ret = coda_command_sync(ctx, CODA_COMMAND_SEQ_INIT);
1314         if (ret < 0) {
1315                 v4l2_err(v4l2_dev, "CODA_COMMAND_SEQ_INIT timeout\n");
1316                 goto out;
1317         }
1318
1319         if (coda_read(dev, CODA_RET_ENC_SEQ_SUCCESS) == 0) {
1320                 v4l2_err(v4l2_dev, "CODA_COMMAND_SEQ_INIT failed\n");
1321                 ret = -EFAULT;
1322                 goto out;
1323         }
1324         ctx->initialized = 1;
1325
1326         if (dst_fourcc != V4L2_PIX_FMT_JPEG) {
1327                 if (dev->devtype->product == CODA_960)
1328                         ctx->num_internal_frames = 4;
1329                 else
1330                         ctx->num_internal_frames = 2;
1331                 ret = coda_alloc_framebuffers(ctx, q_data_src, dst_fourcc);
1332                 if (ret < 0) {
1333                         v4l2_err(v4l2_dev, "failed to allocate framebuffers\n");
1334                         goto out;
1335                 }
1336                 num_fb = 2;
1337                 stride = q_data_src->bytesperline;
1338         } else {
1339                 ctx->num_internal_frames = 0;
1340                 num_fb = 0;
1341                 stride = 0;
1342         }
1343         coda_write(dev, num_fb, CODA_CMD_SET_FRAME_BUF_NUM);
1344         coda_write(dev, stride, CODA_CMD_SET_FRAME_BUF_STRIDE);
1345
1346         if (dev->devtype->product == CODA_HX4 ||
1347             dev->devtype->product == CODA_7541) {
1348                 coda_write(dev, q_data_src->bytesperline,
1349                                 CODA7_CMD_SET_FRAME_SOURCE_BUF_STRIDE);
1350         }
1351         if (dev->devtype->product != CODA_DX6) {
1352                 coda_write(dev, ctx->iram_info.buf_bit_use,
1353                                 CODA7_CMD_SET_FRAME_AXI_BIT_ADDR);
1354                 coda_write(dev, ctx->iram_info.buf_ip_ac_dc_use,
1355                                 CODA7_CMD_SET_FRAME_AXI_IPACDC_ADDR);
1356                 coda_write(dev, ctx->iram_info.buf_dbk_y_use,
1357                                 CODA7_CMD_SET_FRAME_AXI_DBKY_ADDR);
1358                 coda_write(dev, ctx->iram_info.buf_dbk_c_use,
1359                                 CODA7_CMD_SET_FRAME_AXI_DBKC_ADDR);
1360                 coda_write(dev, ctx->iram_info.buf_ovl_use,
1361                                 CODA7_CMD_SET_FRAME_AXI_OVL_ADDR);
1362                 if (dev->devtype->product == CODA_960) {
1363                         coda_write(dev, ctx->iram_info.buf_btp_use,
1364                                         CODA9_CMD_SET_FRAME_AXI_BTP_ADDR);
1365
1366                         coda9_set_frame_cache(ctx, q_data_src->fourcc);
1367
1368                         /* FIXME */
1369                         coda_write(dev, ctx->internal_frames[2].buf.paddr,
1370                                    CODA9_CMD_SET_FRAME_SUBSAMP_A);
1371                         coda_write(dev, ctx->internal_frames[3].buf.paddr,
1372                                    CODA9_CMD_SET_FRAME_SUBSAMP_B);
1373                 }
1374         }
1375
1376         ret = coda_command_sync(ctx, CODA_COMMAND_SET_FRAME_BUF);
1377         if (ret < 0) {
1378                 v4l2_err(v4l2_dev, "CODA_COMMAND_SET_FRAME_BUF timeout\n");
1379                 goto out;
1380         }
1381
1382         coda_dbg(1, ctx, "start encoding %dx%d %4.4s->%4.4s @ %d/%d Hz\n",
1383                  q_data_src->rect.width, q_data_src->rect.height,
1384                  (char *)&ctx->codec->src_fourcc, (char *)&dst_fourcc,
1385                  ctx->params.framerate & 0xffff,
1386                  (ctx->params.framerate >> 16) + 1);
1387
1388         /* Save stream headers */
1389         buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
1390         switch (dst_fourcc) {
1391         case V4L2_PIX_FMT_H264:
1392                 /*
1393                  * Get SPS in the first frame and copy it to an
1394                  * intermediate buffer.
1395                  */
1396                 ret = coda_encode_header(ctx, buf, CODA_HEADER_H264_SPS,
1397                                          &ctx->vpu_header[0][0],
1398                                          &ctx->vpu_header_size[0]);
1399                 if (ret < 0)
1400                         goto out;
1401
1402                 /*
1403                  * If visible width or height are not aligned to macroblock
1404                  * size, the crop_right and crop_bottom SPS fields must be set
1405                  * to the difference between visible and coded size.  This is
1406                  * only supported by CODA960 firmware. All others do not allow
1407                  * writing frame cropping parameters, so we have to manually
1408                  * fix up the SPS RBSP (Sequence Parameter Set Raw Byte
1409                  * Sequence Payload) ourselves.
1410                  */
1411                 if (ctx->dev->devtype->product != CODA_960 &&
1412                     ((q_data_src->rect.width % 16) ||
1413                      (q_data_src->rect.height % 16))) {
1414                         ret = coda_h264_sps_fixup(ctx, q_data_src->rect.width,
1415                                                   q_data_src->rect.height,
1416                                                   &ctx->vpu_header[0][0],
1417                                                   &ctx->vpu_header_size[0],
1418                                                   sizeof(ctx->vpu_header[0]));
1419                         if (ret < 0)
1420                                 goto out;
1421                 }
1422
1423                 /*
1424                  * Get PPS in the first frame and copy it to an
1425                  * intermediate buffer.
1426                  */
1427                 ret = coda_encode_header(ctx, buf, CODA_HEADER_H264_PPS,
1428                                          &ctx->vpu_header[1][0],
1429                                          &ctx->vpu_header_size[1]);
1430                 if (ret < 0)
1431                         goto out;
1432
1433                 /*
1434                  * Length of H.264 headers is variable and thus it might not be
1435                  * aligned for the coda to append the encoded frame. In that is
1436                  * the case a filler NAL must be added to header 2.
1437                  */
1438                 ctx->vpu_header_size[2] = coda_h264_padding(
1439                                         (ctx->vpu_header_size[0] +
1440                                          ctx->vpu_header_size[1]),
1441                                          ctx->vpu_header[2]);
1442                 break;
1443         case V4L2_PIX_FMT_MPEG4:
1444                 /*
1445                  * Get VOS in the first frame and copy it to an
1446                  * intermediate buffer
1447                  */
1448                 ret = coda_encode_header(ctx, buf, CODA_HEADER_MP4V_VOS,
1449                                          &ctx->vpu_header[0][0],
1450                                          &ctx->vpu_header_size[0]);
1451                 if (ret < 0)
1452                         goto out;
1453
1454                 ret = coda_encode_header(ctx, buf, CODA_HEADER_MP4V_VIS,
1455                                          &ctx->vpu_header[1][0],
1456                                          &ctx->vpu_header_size[1]);
1457                 if (ret < 0)
1458                         goto out;
1459
1460                 ret = coda_encode_header(ctx, buf, CODA_HEADER_MP4V_VOL,
1461                                          &ctx->vpu_header[2][0],
1462                                          &ctx->vpu_header_size[2]);
1463                 if (ret < 0)
1464                         goto out;
1465                 break;
1466         default:
1467                 /* No more formats need to save headers at the moment */
1468                 break;
1469         }
1470
1471 out:
1472         mutex_unlock(&dev->coda_mutex);
1473         return ret;
1474 }
1475
1476 static int coda_prepare_encode(struct coda_ctx *ctx)
1477 {
1478         struct coda_q_data *q_data_src, *q_data_dst;
1479         struct vb2_v4l2_buffer *src_buf, *dst_buf;
1480         struct coda_dev *dev = ctx->dev;
1481         int force_ipicture;
1482         int quant_param = 0;
1483         u32 pic_stream_buffer_addr, pic_stream_buffer_size;
1484         u32 rot_mode = 0;
1485         u32 dst_fourcc;
1486         u32 reg;
1487         int ret;
1488
1489         ret = coda_enc_param_change(ctx);
1490         if (ret < 0) {
1491                 v4l2_warn(&ctx->dev->v4l2_dev, "parameter change failed: %d\n",
1492                           ret);
1493         }
1494
1495         src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
1496         dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
1497         q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
1498         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
1499         dst_fourcc = q_data_dst->fourcc;
1500
1501         src_buf->sequence = ctx->osequence;
1502         dst_buf->sequence = ctx->osequence;
1503         ctx->osequence++;
1504
1505         force_ipicture = ctx->params.force_ipicture;
1506         if (force_ipicture)
1507                 ctx->params.force_ipicture = false;
1508         else if (ctx->params.gop_size != 0 &&
1509                  (src_buf->sequence % ctx->params.gop_size) == 0)
1510                 force_ipicture = 1;
1511
1512         /*
1513          * Workaround coda firmware BUG that only marks the first
1514          * frame as IDR. This is a problem for some decoders that can't
1515          * recover when a frame is lost.
1516          */
1517         if (!force_ipicture) {
1518                 src_buf->flags |= V4L2_BUF_FLAG_PFRAME;
1519                 src_buf->flags &= ~V4L2_BUF_FLAG_KEYFRAME;
1520         } else {
1521                 src_buf->flags |= V4L2_BUF_FLAG_KEYFRAME;
1522                 src_buf->flags &= ~V4L2_BUF_FLAG_PFRAME;
1523         }
1524
1525         if (dev->devtype->product == CODA_960)
1526                 coda_set_gdi_regs(ctx);
1527
1528         /*
1529          * Copy headers in front of the first frame and forced I frames for
1530          * H.264 only. In MPEG4 they are already copied by the CODA.
1531          */
1532         if (src_buf->sequence == 0 || force_ipicture) {
1533                 pic_stream_buffer_addr =
1534                         vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0) +
1535                         ctx->vpu_header_size[0] +
1536                         ctx->vpu_header_size[1] +
1537                         ctx->vpu_header_size[2];
1538                 pic_stream_buffer_size = q_data_dst->sizeimage -
1539                         ctx->vpu_header_size[0] -
1540                         ctx->vpu_header_size[1] -
1541                         ctx->vpu_header_size[2];
1542                 memcpy(vb2_plane_vaddr(&dst_buf->vb2_buf, 0),
1543                        &ctx->vpu_header[0][0], ctx->vpu_header_size[0]);
1544                 memcpy(vb2_plane_vaddr(&dst_buf->vb2_buf, 0)
1545                         + ctx->vpu_header_size[0], &ctx->vpu_header[1][0],
1546                         ctx->vpu_header_size[1]);
1547                 memcpy(vb2_plane_vaddr(&dst_buf->vb2_buf, 0)
1548                         + ctx->vpu_header_size[0] + ctx->vpu_header_size[1],
1549                         &ctx->vpu_header[2][0], ctx->vpu_header_size[2]);
1550         } else {
1551                 pic_stream_buffer_addr =
1552                         vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0);
1553                 pic_stream_buffer_size = q_data_dst->sizeimage;
1554         }
1555
1556         if (force_ipicture) {
1557                 switch (dst_fourcc) {
1558                 case V4L2_PIX_FMT_H264:
1559                         quant_param = ctx->params.h264_intra_qp;
1560                         break;
1561                 case V4L2_PIX_FMT_MPEG4:
1562                         quant_param = ctx->params.mpeg4_intra_qp;
1563                         break;
1564                 case V4L2_PIX_FMT_JPEG:
1565                         quant_param = 30;
1566                         break;
1567                 default:
1568                         v4l2_warn(&ctx->dev->v4l2_dev,
1569                                 "cannot set intra qp, fmt not supported\n");
1570                         break;
1571                 }
1572         } else {
1573                 switch (dst_fourcc) {
1574                 case V4L2_PIX_FMT_H264:
1575                         quant_param = ctx->params.h264_inter_qp;
1576                         break;
1577                 case V4L2_PIX_FMT_MPEG4:
1578                         quant_param = ctx->params.mpeg4_inter_qp;
1579                         break;
1580                 default:
1581                         v4l2_warn(&ctx->dev->v4l2_dev,
1582                                 "cannot set inter qp, fmt not supported\n");
1583                         break;
1584                 }
1585         }
1586
1587         /* submit */
1588         if (ctx->params.rot_mode)
1589                 rot_mode = CODA_ROT_MIR_ENABLE | ctx->params.rot_mode;
1590         coda_write(dev, rot_mode, CODA_CMD_ENC_PIC_ROT_MODE);
1591         coda_write(dev, quant_param, CODA_CMD_ENC_PIC_QS);
1592
1593         if (dev->devtype->product == CODA_960) {
1594                 coda_write(dev, 4/*FIXME: 0*/, CODA9_CMD_ENC_PIC_SRC_INDEX);
1595                 coda_write(dev, q_data_src->bytesperline,
1596                            CODA9_CMD_ENC_PIC_SRC_STRIDE);
1597                 coda_write(dev, 0, CODA9_CMD_ENC_PIC_SUB_FRAME_SYNC);
1598
1599                 reg = CODA9_CMD_ENC_PIC_SRC_ADDR_Y;
1600         } else {
1601                 reg = CODA_CMD_ENC_PIC_SRC_ADDR_Y;
1602         }
1603         coda_write_base(ctx, q_data_src, src_buf, reg);
1604
1605         coda_write(dev, force_ipicture << 1 & 0x2,
1606                    CODA_CMD_ENC_PIC_OPTION);
1607
1608         coda_write(dev, pic_stream_buffer_addr, CODA_CMD_ENC_PIC_BB_START);
1609         coda_write(dev, pic_stream_buffer_size / 1024,
1610                    CODA_CMD_ENC_PIC_BB_SIZE);
1611
1612         if (!ctx->streamon_out) {
1613                 /* After streamoff on the output side, set stream end flag */
1614                 ctx->bit_stream_param |= CODA_BIT_STREAM_END_FLAG;
1615                 coda_write(dev, ctx->bit_stream_param,
1616                            CODA_REG_BIT_BIT_STREAM_PARAM);
1617         }
1618
1619         if (dev->devtype->product != CODA_DX6)
1620                 coda_write(dev, ctx->iram_info.axi_sram_use,
1621                                 CODA7_REG_BIT_AXI_SRAM_USE);
1622
1623         trace_coda_enc_pic_run(ctx, src_buf);
1624
1625         coda_command_async(ctx, CODA_COMMAND_PIC_RUN);
1626
1627         return 0;
1628 }
1629
1630 static char coda_frame_type_char(u32 flags)
1631 {
1632         return (flags & V4L2_BUF_FLAG_KEYFRAME) ? 'I' :
1633                (flags & V4L2_BUF_FLAG_PFRAME) ? 'P' :
1634                (flags & V4L2_BUF_FLAG_BFRAME) ? 'B' : '?';
1635 }
1636
1637 static void coda_finish_encode(struct coda_ctx *ctx)
1638 {
1639         struct vb2_v4l2_buffer *src_buf, *dst_buf;
1640         struct coda_dev *dev = ctx->dev;
1641         u32 wr_ptr, start_ptr;
1642
1643         if (ctx->aborting)
1644                 return;
1645
1646         /*
1647          * Lock to make sure that an encoder stop command running in parallel
1648          * will either already have marked src_buf as last, or it will wake up
1649          * the capture queue after the buffers are returned.
1650          */
1651         mutex_lock(&ctx->wakeup_mutex);
1652         src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
1653         dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
1654
1655         trace_coda_enc_pic_done(ctx, dst_buf);
1656
1657         /* Get results from the coda */
1658         start_ptr = coda_read(dev, CODA_CMD_ENC_PIC_BB_START);
1659         wr_ptr = coda_read(dev, CODA_REG_BIT_WR_PTR(ctx->reg_idx));
1660
1661         /* Calculate bytesused field */
1662         if (dst_buf->sequence == 0 ||
1663             src_buf->flags & V4L2_BUF_FLAG_KEYFRAME) {
1664                 vb2_set_plane_payload(&dst_buf->vb2_buf, 0, wr_ptr - start_ptr +
1665                                         ctx->vpu_header_size[0] +
1666                                         ctx->vpu_header_size[1] +
1667                                         ctx->vpu_header_size[2]);
1668         } else {
1669                 vb2_set_plane_payload(&dst_buf->vb2_buf, 0, wr_ptr - start_ptr);
1670         }
1671
1672         coda_dbg(1, ctx, "frame size = %u\n", wr_ptr - start_ptr);
1673
1674         coda_read(dev, CODA_RET_ENC_PIC_SLICE_NUM);
1675         coda_read(dev, CODA_RET_ENC_PIC_FLAG);
1676
1677         dst_buf->flags &= ~(V4L2_BUF_FLAG_KEYFRAME |
1678                             V4L2_BUF_FLAG_PFRAME |
1679                             V4L2_BUF_FLAG_LAST);
1680         if (coda_read(dev, CODA_RET_ENC_PIC_TYPE) == 0)
1681                 dst_buf->flags |= V4L2_BUF_FLAG_KEYFRAME;
1682         else
1683                 dst_buf->flags |= V4L2_BUF_FLAG_PFRAME;
1684         dst_buf->flags |= src_buf->flags & V4L2_BUF_FLAG_LAST;
1685
1686         v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, false);
1687
1688         v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
1689
1690         dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
1691         coda_m2m_buf_done(ctx, dst_buf, VB2_BUF_STATE_DONE);
1692         mutex_unlock(&ctx->wakeup_mutex);
1693
1694         ctx->gopcounter--;
1695         if (ctx->gopcounter < 0)
1696                 ctx->gopcounter = ctx->params.gop_size - 1;
1697
1698         coda_dbg(1, ctx, "job finished: encoded %c frame (%d)%s\n",
1699                  coda_frame_type_char(dst_buf->flags), dst_buf->sequence,
1700                  (dst_buf->flags & V4L2_BUF_FLAG_LAST) ? " (last)" : "");
1701 }
1702
1703 static void coda_seq_end_work(struct work_struct *work)
1704 {
1705         struct coda_ctx *ctx = container_of(work, struct coda_ctx, seq_end_work);
1706         struct coda_dev *dev = ctx->dev;
1707
1708         mutex_lock(&ctx->buffer_mutex);
1709         mutex_lock(&dev->coda_mutex);
1710
1711         if (ctx->initialized == 0)
1712                 goto out;
1713
1714         coda_dbg(1, ctx, "%s: sent command 'SEQ_END' to coda\n", __func__);
1715         if (coda_command_sync(ctx, CODA_COMMAND_SEQ_END)) {
1716                 v4l2_err(&dev->v4l2_dev,
1717                          "CODA_COMMAND_SEQ_END failed\n");
1718         }
1719
1720         /*
1721          * FIXME: Sometimes h.264 encoding fails with 8-byte sequences missing
1722          * from the output stream after the h.264 decoder has run. Resetting the
1723          * hardware after the decoder has finished seems to help.
1724          */
1725         if (dev->devtype->product == CODA_960)
1726                 coda_hw_reset(ctx);
1727
1728         kfifo_init(&ctx->bitstream_fifo,
1729                 ctx->bitstream.vaddr, ctx->bitstream.size);
1730
1731         coda_free_framebuffers(ctx);
1732
1733         ctx->initialized = 0;
1734
1735 out:
1736         mutex_unlock(&dev->coda_mutex);
1737         mutex_unlock(&ctx->buffer_mutex);
1738 }
1739
1740 static void coda_bit_release(struct coda_ctx *ctx)
1741 {
1742         mutex_lock(&ctx->buffer_mutex);
1743         coda_free_framebuffers(ctx);
1744         coda_free_context_buffers(ctx);
1745         coda_free_bitstream_buffer(ctx);
1746         mutex_unlock(&ctx->buffer_mutex);
1747 }
1748
1749 const struct coda_context_ops coda_bit_encode_ops = {
1750         .queue_init = coda_encoder_queue_init,
1751         .reqbufs = coda_encoder_reqbufs,
1752         .start_streaming = coda_start_encoding,
1753         .prepare_run = coda_prepare_encode,
1754         .finish_run = coda_finish_encode,
1755         .seq_end_work = coda_seq_end_work,
1756         .release = coda_bit_release,
1757 };
1758
1759 /*
1760  * Decoder context operations
1761  */
1762
1763 static int coda_alloc_bitstream_buffer(struct coda_ctx *ctx,
1764                                        struct coda_q_data *q_data)
1765 {
1766         if (ctx->bitstream.vaddr)
1767                 return 0;
1768
1769         ctx->bitstream.size = roundup_pow_of_two(q_data->sizeimage * 2);
1770         ctx->bitstream.vaddr = dma_alloc_wc(ctx->dev->dev, ctx->bitstream.size,
1771                                             &ctx->bitstream.paddr, GFP_KERNEL);
1772         if (!ctx->bitstream.vaddr) {
1773                 v4l2_err(&ctx->dev->v4l2_dev,
1774                          "failed to allocate bitstream ringbuffer");
1775                 return -ENOMEM;
1776         }
1777         kfifo_init(&ctx->bitstream_fifo,
1778                    ctx->bitstream.vaddr, ctx->bitstream.size);
1779
1780         return 0;
1781 }
1782
1783 static void coda_free_bitstream_buffer(struct coda_ctx *ctx)
1784 {
1785         if (ctx->bitstream.vaddr == NULL)
1786                 return;
1787
1788         dma_free_wc(ctx->dev->dev, ctx->bitstream.size, ctx->bitstream.vaddr,
1789                     ctx->bitstream.paddr);
1790         ctx->bitstream.vaddr = NULL;
1791         kfifo_init(&ctx->bitstream_fifo, NULL, 0);
1792 }
1793
1794 static int coda_decoder_reqbufs(struct coda_ctx *ctx,
1795                                 struct v4l2_requestbuffers *rb)
1796 {
1797         struct coda_q_data *q_data_src;
1798         int ret;
1799
1800         if (rb->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
1801                 return 0;
1802
1803         if (rb->count) {
1804                 q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
1805                 ret = coda_alloc_context_buffers(ctx, q_data_src);
1806                 if (ret < 0)
1807                         return ret;
1808                 ret = coda_alloc_bitstream_buffer(ctx, q_data_src);
1809                 if (ret < 0) {
1810                         coda_free_context_buffers(ctx);
1811                         return ret;
1812                 }
1813         } else {
1814                 coda_free_bitstream_buffer(ctx);
1815                 coda_free_context_buffers(ctx);
1816         }
1817
1818         return 0;
1819 }
1820
1821 static bool coda_reorder_enable(struct coda_ctx *ctx)
1822 {
1823         struct coda_dev *dev = ctx->dev;
1824         int profile;
1825
1826         if (dev->devtype->product != CODA_HX4 &&
1827             dev->devtype->product != CODA_7541 &&
1828             dev->devtype->product != CODA_960)
1829                 return false;
1830
1831         if (ctx->codec->src_fourcc == V4L2_PIX_FMT_JPEG)
1832                 return false;
1833
1834         if (ctx->codec->src_fourcc != V4L2_PIX_FMT_H264)
1835                 return true;
1836
1837         profile = coda_h264_profile(ctx->params.h264_profile_idc);
1838         if (profile < 0)
1839                 v4l2_warn(&dev->v4l2_dev, "Unknown H264 Profile: %u\n",
1840                           ctx->params.h264_profile_idc);
1841
1842         /* Baseline profile does not support reordering */
1843         return profile > V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE;
1844 }
1845
1846 static int __coda_decoder_seq_init(struct coda_ctx *ctx)
1847 {
1848         struct coda_q_data *q_data_src, *q_data_dst;
1849         u32 bitstream_buf, bitstream_size;
1850         struct coda_dev *dev = ctx->dev;
1851         int width, height;
1852         u32 src_fourcc, dst_fourcc;
1853         u32 val;
1854         int ret;
1855
1856         lockdep_assert_held(&dev->coda_mutex);
1857
1858         coda_dbg(1, ctx, "Video Data Order Adapter: %s\n",
1859                  ctx->use_vdoa ? "Enabled" : "Disabled");
1860
1861         /* Start decoding */
1862         q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
1863         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
1864         bitstream_buf = ctx->bitstream.paddr;
1865         bitstream_size = ctx->bitstream.size;
1866         src_fourcc = q_data_src->fourcc;
1867         dst_fourcc = q_data_dst->fourcc;
1868
1869         /* Update coda bitstream read and write pointers from kfifo */
1870         coda_kfifo_sync_to_device_full(ctx);
1871
1872         ctx->frame_mem_ctrl &= ~(CODA_FRAME_CHROMA_INTERLEAVE | (0x3 << 9) |
1873                                  CODA9_FRAME_TILED2LINEAR);
1874         if (dst_fourcc == V4L2_PIX_FMT_NV12 || dst_fourcc == V4L2_PIX_FMT_YUYV)
1875                 ctx->frame_mem_ctrl |= CODA_FRAME_CHROMA_INTERLEAVE;
1876         if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP)
1877                 ctx->frame_mem_ctrl |= (0x3 << 9) |
1878                         ((ctx->use_vdoa) ? 0 : CODA9_FRAME_TILED2LINEAR);
1879         coda_write(dev, ctx->frame_mem_ctrl, CODA_REG_BIT_FRAME_MEM_CTRL);
1880
1881         ctx->display_idx = -1;
1882         ctx->frm_dis_flg = 0;
1883         coda_write(dev, 0, CODA_REG_BIT_FRM_DIS_FLG(ctx->reg_idx));
1884
1885         coda_write(dev, bitstream_buf, CODA_CMD_DEC_SEQ_BB_START);
1886         coda_write(dev, bitstream_size / 1024, CODA_CMD_DEC_SEQ_BB_SIZE);
1887         val = 0;
1888         if (coda_reorder_enable(ctx))
1889                 val |= CODA_REORDER_ENABLE;
1890         if (ctx->codec->src_fourcc == V4L2_PIX_FMT_JPEG)
1891                 val |= CODA_NO_INT_ENABLE;
1892         coda_write(dev, val, CODA_CMD_DEC_SEQ_OPTION);
1893
1894         ctx->params.codec_mode = ctx->codec->mode;
1895         if (dev->devtype->product == CODA_960 &&
1896             src_fourcc == V4L2_PIX_FMT_MPEG4)
1897                 ctx->params.codec_mode_aux = CODA_MP4_AUX_MPEG4;
1898         else
1899                 ctx->params.codec_mode_aux = 0;
1900         if (src_fourcc == V4L2_PIX_FMT_MPEG4) {
1901                 coda_write(dev, CODA_MP4_CLASS_MPEG4,
1902                            CODA_CMD_DEC_SEQ_MP4_ASP_CLASS);
1903         }
1904         if (src_fourcc == V4L2_PIX_FMT_H264) {
1905                 if (dev->devtype->product == CODA_HX4 ||
1906                     dev->devtype->product == CODA_7541) {
1907                         coda_write(dev, ctx->psbuf.paddr,
1908                                         CODA_CMD_DEC_SEQ_PS_BB_START);
1909                         coda_write(dev, (CODA7_PS_BUF_SIZE / 1024),
1910                                         CODA_CMD_DEC_SEQ_PS_BB_SIZE);
1911                 }
1912                 if (dev->devtype->product == CODA_960) {
1913                         coda_write(dev, 0, CODA_CMD_DEC_SEQ_X264_MV_EN);
1914                         coda_write(dev, 512, CODA_CMD_DEC_SEQ_SPP_CHUNK_SIZE);
1915                 }
1916         }
1917         if (src_fourcc == V4L2_PIX_FMT_JPEG)
1918                 coda_write(dev, 0, CODA_CMD_DEC_SEQ_JPG_THUMB_EN);
1919         if (dev->devtype->product != CODA_960)
1920                 coda_write(dev, 0, CODA_CMD_DEC_SEQ_SRC_SIZE);
1921
1922         ctx->bit_stream_param = CODA_BIT_DEC_SEQ_INIT_ESCAPE;
1923         ret = coda_command_sync(ctx, CODA_COMMAND_SEQ_INIT);
1924         ctx->bit_stream_param = 0;
1925         if (ret) {
1926                 v4l2_err(&dev->v4l2_dev, "CODA_COMMAND_SEQ_INIT timeout\n");
1927                 return ret;
1928         }
1929         ctx->sequence_offset = ~0U;
1930         ctx->initialized = 1;
1931
1932         /* Update kfifo out pointer from coda bitstream read pointer */
1933         coda_kfifo_sync_from_device(ctx);
1934
1935         if (coda_read(dev, CODA_RET_DEC_SEQ_SUCCESS) == 0) {
1936                 v4l2_err(&dev->v4l2_dev,
1937                         "CODA_COMMAND_SEQ_INIT failed, error code = 0x%x\n",
1938                         coda_read(dev, CODA_RET_DEC_SEQ_ERR_REASON));
1939                 return -EAGAIN;
1940         }
1941
1942         val = coda_read(dev, CODA_RET_DEC_SEQ_SRC_SIZE);
1943         if (dev->devtype->product == CODA_DX6) {
1944                 width = (val >> CODADX6_PICWIDTH_OFFSET) & CODADX6_PICWIDTH_MASK;
1945                 height = val & CODADX6_PICHEIGHT_MASK;
1946         } else {
1947                 width = (val >> CODA7_PICWIDTH_OFFSET) & CODA7_PICWIDTH_MASK;
1948                 height = val & CODA7_PICHEIGHT_MASK;
1949         }
1950
1951         if (width > q_data_dst->bytesperline || height > q_data_dst->height) {
1952                 v4l2_err(&dev->v4l2_dev, "stream is %dx%d, not %dx%d\n",
1953                          width, height, q_data_dst->bytesperline,
1954                          q_data_dst->height);
1955                 return -EINVAL;
1956         }
1957
1958         width = round_up(width, 16);
1959         height = round_up(height, 16);
1960
1961         coda_dbg(1, ctx, "start decoding: %dx%d\n", width, height);
1962
1963         ctx->num_internal_frames = coda_read(dev, CODA_RET_DEC_SEQ_FRAME_NEED);
1964         /*
1965          * If the VDOA is used, the decoder needs one additional frame,
1966          * because the frames are freed when the next frame is decoded.
1967          * Otherwise there are visible errors in the decoded frames (green
1968          * regions in displayed frames) and a broken order of frames (earlier
1969          * frames are sporadically displayed after later frames).
1970          */
1971         if (ctx->use_vdoa)
1972                 ctx->num_internal_frames += 1;
1973         if (ctx->num_internal_frames > CODA_MAX_FRAMEBUFFERS) {
1974                 v4l2_err(&dev->v4l2_dev,
1975                          "not enough framebuffers to decode (%d < %d)\n",
1976                          CODA_MAX_FRAMEBUFFERS, ctx->num_internal_frames);
1977                 return -EINVAL;
1978         }
1979
1980         if (src_fourcc == V4L2_PIX_FMT_H264) {
1981                 u32 left_right;
1982                 u32 top_bottom;
1983
1984                 left_right = coda_read(dev, CODA_RET_DEC_SEQ_CROP_LEFT_RIGHT);
1985                 top_bottom = coda_read(dev, CODA_RET_DEC_SEQ_CROP_TOP_BOTTOM);
1986
1987                 q_data_dst->rect.left = (left_right >> 10) & 0x3ff;
1988                 q_data_dst->rect.top = (top_bottom >> 10) & 0x3ff;
1989                 q_data_dst->rect.width = width - q_data_dst->rect.left -
1990                                          (left_right & 0x3ff);
1991                 q_data_dst->rect.height = height - q_data_dst->rect.top -
1992                                           (top_bottom & 0x3ff);
1993         }
1994
1995         if (dev->devtype->product != CODA_DX6) {
1996                 u8 profile, level;
1997
1998                 val = coda_read(dev, CODA7_RET_DEC_SEQ_HEADER_REPORT);
1999                 profile = val & 0xff;
2000                 level = (val >> 8) & 0x7f;
2001
2002                 if (profile || level)
2003                         coda_update_profile_level_ctrls(ctx, profile, level);
2004         }
2005
2006         return 0;
2007 }
2008
2009 static void coda_dec_seq_init_work(struct work_struct *work)
2010 {
2011         struct coda_ctx *ctx = container_of(work,
2012                                             struct coda_ctx, seq_init_work);
2013         struct coda_dev *dev = ctx->dev;
2014         int ret;
2015
2016         mutex_lock(&ctx->buffer_mutex);
2017         mutex_lock(&dev->coda_mutex);
2018
2019         if (ctx->initialized == 1)
2020                 goto out;
2021
2022         ret = __coda_decoder_seq_init(ctx);
2023         if (ret < 0)
2024                 goto out;
2025
2026         ctx->initialized = 1;
2027
2028 out:
2029         mutex_unlock(&dev->coda_mutex);
2030         mutex_unlock(&ctx->buffer_mutex);
2031 }
2032
2033 static int __coda_start_decoding(struct coda_ctx *ctx)
2034 {
2035         struct coda_q_data *q_data_src, *q_data_dst;
2036         struct coda_dev *dev = ctx->dev;
2037         u32 src_fourcc, dst_fourcc;
2038         int ret;
2039
2040         q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
2041         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
2042         src_fourcc = q_data_src->fourcc;
2043         dst_fourcc = q_data_dst->fourcc;
2044
2045         if (!ctx->initialized) {
2046                 ret = __coda_decoder_seq_init(ctx);
2047                 if (ret < 0)
2048                         return ret;
2049         } else {
2050                 ctx->frame_mem_ctrl &= ~(CODA_FRAME_CHROMA_INTERLEAVE | (0x3 << 9) |
2051                                          CODA9_FRAME_TILED2LINEAR);
2052                 if (dst_fourcc == V4L2_PIX_FMT_NV12 || dst_fourcc == V4L2_PIX_FMT_YUYV)
2053                         ctx->frame_mem_ctrl |= CODA_FRAME_CHROMA_INTERLEAVE;
2054                 if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP)
2055                         ctx->frame_mem_ctrl |= (0x3 << 9) |
2056                                 ((ctx->use_vdoa) ? 0 : CODA9_FRAME_TILED2LINEAR);
2057         }
2058
2059         coda_write(dev, ctx->parabuf.paddr, CODA_REG_BIT_PARA_BUF_ADDR);
2060
2061         ret = coda_alloc_framebuffers(ctx, q_data_dst, src_fourcc);
2062         if (ret < 0) {
2063                 v4l2_err(&dev->v4l2_dev, "failed to allocate framebuffers\n");
2064                 return ret;
2065         }
2066
2067         /* Tell the decoder how many frame buffers we allocated. */
2068         coda_write(dev, ctx->num_internal_frames, CODA_CMD_SET_FRAME_BUF_NUM);
2069         coda_write(dev, round_up(q_data_dst->rect.width, 16),
2070                    CODA_CMD_SET_FRAME_BUF_STRIDE);
2071
2072         if (dev->devtype->product != CODA_DX6) {
2073                 /* Set secondary AXI IRAM */
2074                 coda_setup_iram(ctx);
2075
2076                 coda_write(dev, ctx->iram_info.buf_bit_use,
2077                                 CODA7_CMD_SET_FRAME_AXI_BIT_ADDR);
2078                 coda_write(dev, ctx->iram_info.buf_ip_ac_dc_use,
2079                                 CODA7_CMD_SET_FRAME_AXI_IPACDC_ADDR);
2080                 coda_write(dev, ctx->iram_info.buf_dbk_y_use,
2081                                 CODA7_CMD_SET_FRAME_AXI_DBKY_ADDR);
2082                 coda_write(dev, ctx->iram_info.buf_dbk_c_use,
2083                                 CODA7_CMD_SET_FRAME_AXI_DBKC_ADDR);
2084                 coda_write(dev, ctx->iram_info.buf_ovl_use,
2085                                 CODA7_CMD_SET_FRAME_AXI_OVL_ADDR);
2086                 if (dev->devtype->product == CODA_960) {
2087                         coda_write(dev, ctx->iram_info.buf_btp_use,
2088                                         CODA9_CMD_SET_FRAME_AXI_BTP_ADDR);
2089
2090                         coda_write(dev, -1, CODA9_CMD_SET_FRAME_DELAY);
2091                         coda9_set_frame_cache(ctx, dst_fourcc);
2092                 }
2093         }
2094
2095         if (src_fourcc == V4L2_PIX_FMT_H264) {
2096                 coda_write(dev, ctx->slicebuf.paddr,
2097                                 CODA_CMD_SET_FRAME_SLICE_BB_START);
2098                 coda_write(dev, ctx->slicebuf.size / 1024,
2099                                 CODA_CMD_SET_FRAME_SLICE_BB_SIZE);
2100         }
2101
2102         if (dev->devtype->product == CODA_HX4 ||
2103             dev->devtype->product == CODA_7541) {
2104                 int max_mb_x = 1920 / 16;
2105                 int max_mb_y = 1088 / 16;
2106                 int max_mb_num = max_mb_x * max_mb_y;
2107
2108                 coda_write(dev, max_mb_num << 16 | max_mb_x << 8 | max_mb_y,
2109                                 CODA7_CMD_SET_FRAME_MAX_DEC_SIZE);
2110         } else if (dev->devtype->product == CODA_960) {
2111                 int max_mb_x = 1920 / 16;
2112                 int max_mb_y = 1088 / 16;
2113                 int max_mb_num = max_mb_x * max_mb_y;
2114
2115                 coda_write(dev, max_mb_num << 16 | max_mb_x << 8 | max_mb_y,
2116                                 CODA9_CMD_SET_FRAME_MAX_DEC_SIZE);
2117         }
2118
2119         if (coda_command_sync(ctx, CODA_COMMAND_SET_FRAME_BUF)) {
2120                 v4l2_err(&ctx->dev->v4l2_dev,
2121                          "CODA_COMMAND_SET_FRAME_BUF timeout\n");
2122                 return -ETIMEDOUT;
2123         }
2124
2125         return 0;
2126 }
2127
2128 static int coda_start_decoding(struct coda_ctx *ctx)
2129 {
2130         struct coda_dev *dev = ctx->dev;
2131         int ret;
2132
2133         mutex_lock(&dev->coda_mutex);
2134         ret = __coda_start_decoding(ctx);
2135         mutex_unlock(&dev->coda_mutex);
2136
2137         return ret;
2138 }
2139
2140 static int coda_prepare_decode(struct coda_ctx *ctx)
2141 {
2142         struct vb2_v4l2_buffer *dst_buf;
2143         struct coda_dev *dev = ctx->dev;
2144         struct coda_q_data *q_data_dst;
2145         struct coda_buffer_meta *meta;
2146         u32 rot_mode = 0;
2147         u32 reg_addr, reg_stride;
2148
2149         dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
2150         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
2151
2152         /* Try to copy source buffer contents into the bitstream ringbuffer */
2153         mutex_lock(&ctx->bitstream_mutex);
2154         coda_fill_bitstream(ctx, NULL);
2155         mutex_unlock(&ctx->bitstream_mutex);
2156
2157         if (coda_get_bitstream_payload(ctx) < 512 &&
2158             (!(ctx->bit_stream_param & CODA_BIT_STREAM_END_FLAG))) {
2159                 coda_dbg(1, ctx, "bitstream payload: %d, skipping\n",
2160                          coda_get_bitstream_payload(ctx));
2161                 v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
2162                 return -EAGAIN;
2163         }
2164
2165         /* Run coda_start_decoding (again) if not yet initialized */
2166         if (!ctx->initialized) {
2167                 int ret = __coda_start_decoding(ctx);
2168
2169                 if (ret < 0) {
2170                         v4l2_err(&dev->v4l2_dev, "failed to start decoding\n");
2171                         v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
2172                         return -EAGAIN;
2173                 } else {
2174                         ctx->initialized = 1;
2175                 }
2176         }
2177
2178         if (dev->devtype->product == CODA_960)
2179                 coda_set_gdi_regs(ctx);
2180
2181         if (ctx->use_vdoa &&
2182             ctx->display_idx >= 0 &&
2183             ctx->display_idx < ctx->num_internal_frames) {
2184                 vdoa_device_run(ctx->vdoa,
2185                                 vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0),
2186                                 ctx->internal_frames[ctx->display_idx].buf.paddr);
2187         } else {
2188                 if (dev->devtype->product == CODA_960) {
2189                         /*
2190                          * It was previously assumed that the CODA960 has an
2191                          * internal list of 64 buffer entries that contains
2192                          * both the registered internal frame buffers as well
2193                          * as the rotator buffer output, and that the ROT_INDEX
2194                          * register must be set to a value between the last
2195                          * internal frame buffers' index and 64.
2196                          * At least on firmware version 3.1.1 it turns out that
2197                          * setting ROT_INDEX to any value >= 32 causes CODA
2198                          * hangups that it can not recover from with the SRC VPU
2199                          * reset.
2200                          * It does appear to work however, to just set it to a
2201                          * fixed value in the [ctx->num_internal_frames, 31]
2202                          * range, for example CODA_MAX_FRAMEBUFFERS.
2203                          */
2204                         coda_write(dev, CODA_MAX_FRAMEBUFFERS,
2205                                    CODA9_CMD_DEC_PIC_ROT_INDEX);
2206
2207                         reg_addr = CODA9_CMD_DEC_PIC_ROT_ADDR_Y;
2208                         reg_stride = CODA9_CMD_DEC_PIC_ROT_STRIDE;
2209                 } else {
2210                         reg_addr = CODA_CMD_DEC_PIC_ROT_ADDR_Y;
2211                         reg_stride = CODA_CMD_DEC_PIC_ROT_STRIDE;
2212                 }
2213                 coda_write_base(ctx, q_data_dst, dst_buf, reg_addr);
2214                 coda_write(dev, q_data_dst->bytesperline, reg_stride);
2215
2216                 rot_mode = CODA_ROT_MIR_ENABLE | ctx->params.rot_mode;
2217         }
2218
2219         coda_write(dev, rot_mode, CODA_CMD_DEC_PIC_ROT_MODE);
2220
2221         switch (dev->devtype->product) {
2222         case CODA_DX6:
2223                 /* TBD */
2224         case CODA_HX4:
2225         case CODA_7541:
2226                 coda_write(dev, CODA_PRE_SCAN_EN, CODA_CMD_DEC_PIC_OPTION);
2227                 break;
2228         case CODA_960:
2229                 /* 'hardcode to use interrupt disable mode'? */
2230                 coda_write(dev, (1 << 10), CODA_CMD_DEC_PIC_OPTION);
2231                 break;
2232         }
2233
2234         coda_write(dev, 0, CODA_CMD_DEC_PIC_SKIP_NUM);
2235
2236         coda_write(dev, 0, CODA_CMD_DEC_PIC_BB_START);
2237         coda_write(dev, 0, CODA_CMD_DEC_PIC_START_BYTE);
2238
2239         if (dev->devtype->product != CODA_DX6)
2240                 coda_write(dev, ctx->iram_info.axi_sram_use,
2241                                 CODA7_REG_BIT_AXI_SRAM_USE);
2242
2243         spin_lock(&ctx->buffer_meta_lock);
2244         meta = list_first_entry_or_null(&ctx->buffer_meta_list,
2245                                         struct coda_buffer_meta, list);
2246
2247         if (meta && ctx->codec->src_fourcc == V4L2_PIX_FMT_JPEG) {
2248
2249                 /* If this is the last buffer in the bitstream, add padding */
2250                 if (meta->end == ctx->bitstream_fifo.kfifo.in) {
2251                         static unsigned char buf[512];
2252                         unsigned int pad;
2253
2254                         /* Pad to multiple of 256 and then add 256 more */
2255                         pad = ((0 - meta->end) & 0xff) + 256;
2256
2257                         memset(buf, 0xff, sizeof(buf));
2258
2259                         kfifo_in(&ctx->bitstream_fifo, buf, pad);
2260                 }
2261         }
2262         spin_unlock(&ctx->buffer_meta_lock);
2263
2264         coda_kfifo_sync_to_device_full(ctx);
2265
2266         /* Clear decode success flag */
2267         coda_write(dev, 0, CODA_RET_DEC_PIC_SUCCESS);
2268
2269         /* Clear error return value */
2270         coda_write(dev, 0, CODA_RET_DEC_PIC_ERR_MB);
2271
2272         trace_coda_dec_pic_run(ctx, meta);
2273
2274         coda_command_async(ctx, CODA_COMMAND_PIC_RUN);
2275
2276         return 0;
2277 }
2278
2279 static void coda_finish_decode(struct coda_ctx *ctx)
2280 {
2281         struct coda_dev *dev = ctx->dev;
2282         struct coda_q_data *q_data_src;
2283         struct coda_q_data *q_data_dst;
2284         struct vb2_v4l2_buffer *dst_buf;
2285         struct coda_buffer_meta *meta;
2286         int width, height;
2287         int decoded_idx;
2288         int display_idx;
2289         struct coda_internal_frame *decoded_frame = NULL;
2290         u32 src_fourcc;
2291         int success;
2292         u32 err_mb;
2293         int err_vdoa = 0;
2294         u32 val;
2295
2296         if (ctx->aborting)
2297                 return;
2298
2299         /* Update kfifo out pointer from coda bitstream read pointer */
2300         coda_kfifo_sync_from_device(ctx);
2301
2302         /*
2303          * in stream-end mode, the read pointer can overshoot the write pointer
2304          * by up to 512 bytes
2305          */
2306         if (ctx->bit_stream_param & CODA_BIT_STREAM_END_FLAG) {
2307                 if (coda_get_bitstream_payload(ctx) >= ctx->bitstream.size - 512)
2308                         kfifo_init(&ctx->bitstream_fifo,
2309                                 ctx->bitstream.vaddr, ctx->bitstream.size);
2310         }
2311
2312         q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
2313         src_fourcc = q_data_src->fourcc;
2314
2315         val = coda_read(dev, CODA_RET_DEC_PIC_SUCCESS);
2316         if (val != 1)
2317                 pr_err("DEC_PIC_SUCCESS = %d\n", val);
2318
2319         success = val & 0x1;
2320         if (!success)
2321                 v4l2_err(&dev->v4l2_dev, "decode failed\n");
2322
2323         if (src_fourcc == V4L2_PIX_FMT_H264) {
2324                 if (val & (1 << 3))
2325                         v4l2_err(&dev->v4l2_dev,
2326                                  "insufficient PS buffer space (%d bytes)\n",
2327                                  ctx->psbuf.size);
2328                 if (val & (1 << 2))
2329                         v4l2_err(&dev->v4l2_dev,
2330                                  "insufficient slice buffer space (%d bytes)\n",
2331                                  ctx->slicebuf.size);
2332         }
2333
2334         val = coda_read(dev, CODA_RET_DEC_PIC_SIZE);
2335         width = (val >> 16) & 0xffff;
2336         height = val & 0xffff;
2337
2338         q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
2339
2340         /* frame crop information */
2341         if (src_fourcc == V4L2_PIX_FMT_H264) {
2342                 u32 left_right;
2343                 u32 top_bottom;
2344
2345                 left_right = coda_read(dev, CODA_RET_DEC_PIC_CROP_LEFT_RIGHT);
2346                 top_bottom = coda_read(dev, CODA_RET_DEC_PIC_CROP_TOP_BOTTOM);
2347
2348                 if (left_right == 0xffffffff && top_bottom == 0xffffffff) {
2349                         /* Keep current crop information */
2350                 } else {
2351                         struct v4l2_rect *rect = &q_data_dst->rect;
2352
2353                         rect->left = left_right >> 16 & 0xffff;
2354                         rect->top = top_bottom >> 16 & 0xffff;
2355                         rect->width = width - rect->left -
2356                                       (left_right & 0xffff);
2357                         rect->height = height - rect->top -
2358                                        (top_bottom & 0xffff);
2359                 }
2360         } else {
2361                 /* no cropping */
2362         }
2363
2364         err_mb = coda_read(dev, CODA_RET_DEC_PIC_ERR_MB);
2365         if (err_mb > 0)
2366                 v4l2_err(&dev->v4l2_dev,
2367                          "errors in %d macroblocks\n", err_mb);
2368
2369         if (dev->devtype->product == CODA_HX4 ||
2370             dev->devtype->product == CODA_7541) {
2371                 val = coda_read(dev, CODA_RET_DEC_PIC_OPTION);
2372                 if (val == 0) {
2373                         /* not enough bitstream data */
2374                         coda_dbg(1, ctx, "prescan failed: %d\n", val);
2375                         ctx->hold = true;
2376                         return;
2377                 }
2378         }
2379
2380         /* Wait until the VDOA finished writing the previous display frame */
2381         if (ctx->use_vdoa &&
2382             ctx->display_idx >= 0 &&
2383             ctx->display_idx < ctx->num_internal_frames) {
2384                 err_vdoa = vdoa_wait_for_completion(ctx->vdoa);
2385         }
2386
2387         ctx->frm_dis_flg = coda_read(dev,
2388                                      CODA_REG_BIT_FRM_DIS_FLG(ctx->reg_idx));
2389
2390         /* The previous display frame was copied out and can be overwritten */
2391         if (ctx->display_idx >= 0 &&
2392             ctx->display_idx < ctx->num_internal_frames) {
2393                 ctx->frm_dis_flg &= ~(1 << ctx->display_idx);
2394                 coda_write(dev, ctx->frm_dis_flg,
2395                                 CODA_REG_BIT_FRM_DIS_FLG(ctx->reg_idx));
2396         }
2397
2398         /*
2399          * The index of the last decoded frame, not necessarily in
2400          * display order, and the index of the next display frame.
2401          * The latter could have been decoded in a previous run.
2402          */
2403         decoded_idx = coda_read(dev, CODA_RET_DEC_PIC_CUR_IDX);
2404         display_idx = coda_read(dev, CODA_RET_DEC_PIC_FRAME_IDX);
2405
2406         if (decoded_idx == -1) {
2407                 /* no frame was decoded, but we might have a display frame */
2408                 if (display_idx >= 0 && display_idx < ctx->num_internal_frames)
2409                         ctx->sequence_offset++;
2410                 else if (ctx->display_idx < 0)
2411                         ctx->hold = true;
2412         } else if (decoded_idx == -2) {
2413                 if (ctx->display_idx >= 0 &&
2414                     ctx->display_idx < ctx->num_internal_frames)
2415                         ctx->sequence_offset++;
2416                 /* no frame was decoded, we still return remaining buffers */
2417         } else if (decoded_idx < 0 || decoded_idx >= ctx->num_internal_frames) {
2418                 v4l2_err(&dev->v4l2_dev,
2419                          "decoded frame index out of range: %d\n", decoded_idx);
2420         } else {
2421                 decoded_frame = &ctx->internal_frames[decoded_idx];
2422
2423                 val = coda_read(dev, CODA_RET_DEC_PIC_FRAME_NUM);
2424                 if (ctx->sequence_offset == -1)
2425                         ctx->sequence_offset = val;
2426                 val -= ctx->sequence_offset;
2427                 spin_lock(&ctx->buffer_meta_lock);
2428                 if (!list_empty(&ctx->buffer_meta_list)) {
2429                         meta = list_first_entry(&ctx->buffer_meta_list,
2430                                               struct coda_buffer_meta, list);
2431                         list_del(&meta->list);
2432                         ctx->num_metas--;
2433                         spin_unlock(&ctx->buffer_meta_lock);
2434                         /*
2435                          * Clamp counters to 16 bits for comparison, as the HW
2436                          * counter rolls over at this point for h.264. This
2437                          * may be different for other formats, but using 16 bits
2438                          * should be enough to detect most errors and saves us
2439                          * from doing different things based on the format.
2440                          */
2441                         if ((val & 0xffff) != (meta->sequence & 0xffff)) {
2442                                 v4l2_err(&dev->v4l2_dev,
2443                                          "sequence number mismatch (%d(%d) != %d)\n",
2444                                          val, ctx->sequence_offset,
2445                                          meta->sequence);
2446                         }
2447                         decoded_frame->meta = *meta;
2448                         kfree(meta);
2449                 } else {
2450                         spin_unlock(&ctx->buffer_meta_lock);
2451                         v4l2_err(&dev->v4l2_dev, "empty timestamp list!\n");
2452                         memset(&decoded_frame->meta, 0,
2453                                sizeof(struct coda_buffer_meta));
2454                         decoded_frame->meta.sequence = val;
2455                         decoded_frame->meta.last = false;
2456                         ctx->sequence_offset++;
2457                 }
2458
2459                 trace_coda_dec_pic_done(ctx, &decoded_frame->meta);
2460
2461                 val = coda_read(dev, CODA_RET_DEC_PIC_TYPE) & 0x7;
2462                 decoded_frame->type = (val == 0) ? V4L2_BUF_FLAG_KEYFRAME :
2463                                       (val == 1) ? V4L2_BUF_FLAG_PFRAME :
2464                                                    V4L2_BUF_FLAG_BFRAME;
2465
2466                 decoded_frame->error = err_mb;
2467         }
2468
2469         if (display_idx == -1) {
2470                 /*
2471                  * no more frames to be decoded, but there could still
2472                  * be rotator output to dequeue
2473                  */
2474                 ctx->hold = true;
2475         } else if (display_idx == -3) {
2476                 /* possibly prescan failure */
2477         } else if (display_idx < 0 || display_idx >= ctx->num_internal_frames) {
2478                 v4l2_err(&dev->v4l2_dev,
2479                          "presentation frame index out of range: %d\n",
2480                          display_idx);
2481         }
2482
2483         /* If a frame was copied out, return it */
2484         if (ctx->display_idx >= 0 &&
2485             ctx->display_idx < ctx->num_internal_frames) {
2486                 struct coda_internal_frame *ready_frame;
2487
2488                 ready_frame = &ctx->internal_frames[ctx->display_idx];
2489
2490                 dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
2491                 dst_buf->sequence = ctx->osequence++;
2492
2493                 dst_buf->field = V4L2_FIELD_NONE;
2494                 dst_buf->flags &= ~(V4L2_BUF_FLAG_KEYFRAME |
2495                                              V4L2_BUF_FLAG_PFRAME |
2496                                              V4L2_BUF_FLAG_BFRAME);
2497                 dst_buf->flags |= ready_frame->type;
2498                 meta = &ready_frame->meta;
2499                 if (meta->last && !coda_reorder_enable(ctx)) {
2500                         /*
2501                          * If this was the last decoded frame, and reordering
2502                          * is disabled, this will be the last display frame.
2503                          */
2504                         coda_dbg(1, ctx, "last meta, marking as last frame\n");
2505                         dst_buf->flags |= V4L2_BUF_FLAG_LAST;
2506                 } else if (ctx->bit_stream_param & CODA_BIT_STREAM_END_FLAG &&
2507                            display_idx == -1) {
2508                         /*
2509                          * If there is no designated presentation frame anymore,
2510                          * this frame has to be the last one.
2511                          */
2512                         coda_dbg(1, ctx,
2513                                  "no more frames to return, marking as last frame\n");
2514                         dst_buf->flags |= V4L2_BUF_FLAG_LAST;
2515                 }
2516                 dst_buf->timecode = meta->timecode;
2517                 dst_buf->vb2_buf.timestamp = meta->timestamp;
2518
2519                 trace_coda_dec_rot_done(ctx, dst_buf, meta);
2520
2521                 vb2_set_plane_payload(&dst_buf->vb2_buf, 0,
2522                                       q_data_dst->sizeimage);
2523
2524                 if (ready_frame->error || err_vdoa)
2525                         coda_m2m_buf_done(ctx, dst_buf, VB2_BUF_STATE_ERROR);
2526                 else
2527                         coda_m2m_buf_done(ctx, dst_buf, VB2_BUF_STATE_DONE);
2528
2529                 if (decoded_frame) {
2530                         coda_dbg(1, ctx, "job finished: decoded %c frame %u, returned %c frame %u (%u/%u)%s\n",
2531                                  coda_frame_type_char(decoded_frame->type),
2532                                  decoded_frame->meta.sequence,
2533                                  coda_frame_type_char(dst_buf->flags),
2534                                  ready_frame->meta.sequence,
2535                                  dst_buf->sequence, ctx->qsequence,
2536                                  (dst_buf->flags & V4L2_BUF_FLAG_LAST) ?
2537                                  " (last)" : "");
2538                 } else {
2539                         coda_dbg(1, ctx, "job finished: no frame decoded (%d), returned %c frame %u (%u/%u)%s\n",
2540                                  decoded_idx,
2541                                  coda_frame_type_char(dst_buf->flags),
2542                                  ready_frame->meta.sequence,
2543                                  dst_buf->sequence, ctx->qsequence,
2544                                  (dst_buf->flags & V4L2_BUF_FLAG_LAST) ?
2545                                  " (last)" : "");
2546                 }
2547         } else {
2548                 if (decoded_frame) {
2549                         coda_dbg(1, ctx, "job finished: decoded %c frame %u, no frame returned (%d)\n",
2550                                  coda_frame_type_char(decoded_frame->type),
2551                                  decoded_frame->meta.sequence,
2552                                  ctx->display_idx);
2553                 } else {
2554                         coda_dbg(1, ctx, "job finished: no frame decoded (%d) or returned (%d)\n",
2555                                  decoded_idx, ctx->display_idx);
2556                 }
2557         }
2558
2559         /* The rotator will copy the current display frame next time */
2560         ctx->display_idx = display_idx;
2561
2562         /*
2563          * The current decode run might have brought the bitstream fill level
2564          * below the size where we can start the next decode run. As userspace
2565          * might have filled the output queue completely and might thus be
2566          * blocked, we can't rely on the next qbuf to trigger the bitstream
2567          * refill. Check if we have data to refill the bitstream now.
2568          */
2569         mutex_lock(&ctx->bitstream_mutex);
2570         coda_fill_bitstream(ctx, NULL);
2571         mutex_unlock(&ctx->bitstream_mutex);
2572 }
2573
2574 static void coda_decode_timeout(struct coda_ctx *ctx)
2575 {
2576         struct vb2_v4l2_buffer *dst_buf;
2577
2578         /*
2579          * For now this only handles the case where we would deadlock with
2580          * userspace, i.e. userspace issued DEC_CMD_STOP and waits for EOS,
2581          * but after a failed decode run we would hold the context and wait for
2582          * userspace to queue more buffers.
2583          */
2584         if (!(ctx->bit_stream_param & CODA_BIT_STREAM_END_FLAG))
2585                 return;
2586
2587         dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
2588         dst_buf->sequence = ctx->qsequence - 1;
2589
2590         coda_m2m_buf_done(ctx, dst_buf, VB2_BUF_STATE_ERROR);
2591 }
2592
2593 const struct coda_context_ops coda_bit_decode_ops = {
2594         .queue_init = coda_decoder_queue_init,
2595         .reqbufs = coda_decoder_reqbufs,
2596         .start_streaming = coda_start_decoding,
2597         .prepare_run = coda_prepare_decode,
2598         .finish_run = coda_finish_decode,
2599         .run_timeout = coda_decode_timeout,
2600         .seq_init_work = coda_dec_seq_init_work,
2601         .seq_end_work = coda_seq_end_work,
2602         .release = coda_bit_release,
2603 };
2604
2605 irqreturn_t coda_irq_handler(int irq, void *data)
2606 {
2607         struct coda_dev *dev = data;
2608         struct coda_ctx *ctx;
2609
2610         /* read status register to attend the IRQ */
2611         coda_read(dev, CODA_REG_BIT_INT_STATUS);
2612         coda_write(dev, 0, CODA_REG_BIT_INT_REASON);
2613         coda_write(dev, CODA_REG_BIT_INT_CLEAR_SET,
2614                       CODA_REG_BIT_INT_CLEAR);
2615
2616         ctx = v4l2_m2m_get_curr_priv(dev->m2m_dev);
2617         if (ctx == NULL) {
2618                 v4l2_err(&dev->v4l2_dev,
2619                          "Instance released before the end of transaction\n");
2620                 return IRQ_HANDLED;
2621         }
2622
2623         trace_coda_bit_done(ctx);
2624
2625         if (ctx->aborting) {
2626                 coda_dbg(1, ctx, "task has been aborted\n");
2627         }
2628
2629         if (coda_isbusy(ctx->dev)) {
2630                 coda_dbg(1, ctx, "coda is still busy!!!!\n");
2631                 return IRQ_NONE;
2632         }
2633
2634         complete(&ctx->completion);
2635
2636         return IRQ_HANDLED;
2637 }