drivers/staging/media/tegra-vde/vde.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * NVIDIA Tegra Video decoder driver
   4  *
   5  * Copyright (C) 2016-2017 Dmitry Osipenko <digetx@gmail.com>
   6  *
   7  */
   8
   9 #include <linux/clk.h>
  10 #include <linux/dma-buf.h>
  11 #include <linux/genalloc.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/iopoll.h>
  14 #include <linux/list.h>
  15 #include <linux/miscdevice.h>
  16 #include <linux/module.h>
  17 #include <linux/of_device.h>
  18 #include <linux/pm_runtime.h>
  19 #include <linux/reset.h>
  20 #include <linux/slab.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <soc/tegra/pmc.h>
  24
  25 #include "uapi.h"
  26 #include "vde.h"
  27
  28 #define CREATE_TRACE_POINTS
  29 #include "trace.h"
  30
  31 #define ICMDQUE_WR              0x00
  32 #define CMDQUE_CONTROL          0x08
  33 #define INTR_STATUS             0x18
  34 #define BSE_INT_ENB             0x40
  35 #define BSE_CONFIG              0x44
  36
  37 #define BSE_ICMDQUE_EMPTY       BIT(3)
  38 #define BSE_DMA_BUSY            BIT(23)
  39
  40 struct video_frame {
  41         struct dma_buf_attachment *y_dmabuf_attachment;
  42         struct dma_buf_attachment *cb_dmabuf_attachment;
  43         struct dma_buf_attachment *cr_dmabuf_attachment;
  44         struct dma_buf_attachment *aux_dmabuf_attachment;
  45         dma_addr_t y_addr;
  46         dma_addr_t cb_addr;
  47         dma_addr_t cr_addr;
  48         dma_addr_t aux_addr;
  49         u32 frame_num;
  50         u32 flags;
  51 };
  52
  53 static void tegra_vde_writel(struct tegra_vde *vde,
  54                              u32 value, void __iomem *base, u32 offset)
  55 {
  56         trace_vde_writel(vde, base, offset, value);
  57
  58         writel_relaxed(value, base + offset);
  59 }
  60
  61 static u32 tegra_vde_readl(struct tegra_vde *vde,
  62                            void __iomem *base, u32 offset)
  63 {
  64         u32 value = readl_relaxed(base + offset);
  65
  66         trace_vde_readl(vde, base, offset, value);
  67
  68         return value;
  69 }
  70
  71 static void tegra_vde_set_bits(struct tegra_vde *vde,
  72                                u32 mask, void __iomem *base, u32 offset)
  73 {
  74         u32 value = tegra_vde_readl(vde, base, offset);
  75
  76         tegra_vde_writel(vde, value | mask, base, offset);
  77 }
  78
  79 static int tegra_vde_wait_mbe(struct tegra_vde *vde)
  80 {
  81         u32 tmp;
  82
  83         return readl_relaxed_poll_timeout(vde->mbe + 0x8C, tmp,
  84                                           (tmp >= 0x10), 1, 100);
  85 }
  86
  87 static int tegra_vde_setup_mbe_frame_idx(struct tegra_vde *vde,
  88                                          unsigned int refs_nb,
  89                                          bool setup_refs)
  90 {
  91         u32 frame_idx_enb_mask = 0;
  92         u32 value;
  93         unsigned int frame_idx;
  94         unsigned int idx;
  95         int err;
  96
  97         tegra_vde_writel(vde, 0xD0000000 | (0 << 23), vde->mbe, 0x80);
  98         tegra_vde_writel(vde, 0xD0200000 | (0 << 23), vde->mbe, 0x80);
  99
 100         err = tegra_vde_wait_mbe(vde);
 101         if (err)
 102                 return err;
 103
 104         if (!setup_refs)
 105                 return 0;
 106
 107         for (idx = 0, frame_idx = 1; idx < refs_nb; idx++, frame_idx++) {
 108                 tegra_vde_writel(vde, 0xD0000000 | (frame_idx << 23),
 109                                  vde->mbe, 0x80);
 110                 tegra_vde_writel(vde, 0xD0200000 | (frame_idx << 23),
 111                                  vde->mbe, 0x80);
 112
 113                 frame_idx_enb_mask |= frame_idx << (6 * (idx % 4));
 114
 115                 if (idx % 4 == 3 || idx == refs_nb - 1) {
 116                         value = 0xC0000000;
 117                         value |= (idx >> 2) << 24;
 118                         value |= frame_idx_enb_mask;
 119
 120                         tegra_vde_writel(vde, value, vde->mbe, 0x80);
 121
 122                         err = tegra_vde_wait_mbe(vde);
 123                         if (err)
 124                                 return err;
 125
 126                         frame_idx_enb_mask = 0;
 127                 }
 128         }
 129
 130         return 0;
 131 }
 132
 133 static void tegra_vde_mbe_set_0xa_reg(struct tegra_vde *vde, int reg, u32 val)
 134 {
 135         tegra_vde_writel(vde, 0xA0000000 | (reg << 24) | (val & 0xFFFF),
 136                          vde->mbe, 0x80);
 137         tegra_vde_writel(vde, 0xA0000000 | ((reg + 1) << 24) | (val >> 16),
 138                          vde->mbe, 0x80);
 139 }
 140
 141 static int tegra_vde_wait_bsev(struct tegra_vde *vde, bool wait_dma)
 142 {
 143         struct device *dev = vde->miscdev.parent;
 144         u32 value;
 145         int err;
 146
 147         err = readl_relaxed_poll_timeout(vde->bsev + INTR_STATUS, value,
 148                                          !(value & BIT(2)), 1, 100);
 149         if (err) {
 150                 dev_err(dev, "BSEV unknown bit timeout\n");
 151                 return err;
 152         }
 153
 154         err = readl_relaxed_poll_timeout(vde->bsev + INTR_STATUS, value,
 155                                          (value & BSE_ICMDQUE_EMPTY), 1, 100);
 156         if (err) {
 157                 dev_err(dev, "BSEV ICMDQUE flush timeout\n");
 158                 return err;
 159         }
 160
 161         if (!wait_dma)
 162                 return 0;
 163
 164         err = readl_relaxed_poll_timeout(vde->bsev + INTR_STATUS, value,
 165                                          !(value & BSE_DMA_BUSY), 1, 100);
 166         if (err) {
 167                 dev_err(dev, "BSEV DMA timeout\n");
 168                 return err;
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int tegra_vde_push_to_bsev_icmdqueue(struct tegra_vde *vde,
 175                                             u32 value, bool wait_dma)
 176 {
 177         tegra_vde_writel(vde, value, vde->bsev, ICMDQUE_WR);
 178
 179         return tegra_vde_wait_bsev(vde, wait_dma);
 180 }
 181
 182 static void tegra_vde_setup_frameid(struct tegra_vde *vde,
 183                                     struct video_frame *frame,
 184                                     unsigned int frameid,
 185                                     u32 mbs_width, u32 mbs_height)
 186 {
 187         u32 y_addr  = frame ? frame->y_addr  : 0x6CDEAD00;
 188         u32 cb_addr = frame ? frame->cb_addr : 0x6CDEAD00;
 189         u32 cr_addr = frame ? frame->cr_addr : 0x6CDEAD00;
 190         u32 value1 = frame ? ((mbs_width << 16) | mbs_height) : 0;
 191         u32 value2 = frame ? ((((mbs_width + 1) >> 1) << 6) | 1) : 0;
 192
 193         tegra_vde_writel(vde, y_addr  >> 8, vde->frameid, 0x000 + frameid * 4);
 194         tegra_vde_writel(vde, cb_addr >> 8, vde->frameid, 0x100 + frameid * 4);
 195         tegra_vde_writel(vde, cr_addr >> 8, vde->frameid, 0x180 + frameid * 4);
 196         tegra_vde_writel(vde, value1,       vde->frameid, 0x080 + frameid * 4);
 197         tegra_vde_writel(vde, value2,       vde->frameid, 0x280 + frameid * 4);
 198 }
 199
 200 static void tegra_setup_frameidx(struct tegra_vde *vde,
 201                                  struct video_frame *frames,
 202                                  unsigned int frames_nb,
 203                                  u32 mbs_width, u32 mbs_height)
 204 {
 205         unsigned int idx;
 206
 207         for (idx = 0; idx < frames_nb; idx++)
 208                 tegra_vde_setup_frameid(vde, &frames[idx], idx,
 209                                         mbs_width, mbs_height);
 210
 211         for (; idx < 17; idx++)
 212                 tegra_vde_setup_frameid(vde, NULL, idx, 0, 0);
 213 }
 214
 215 static void tegra_vde_setup_iram_entry(struct tegra_vde *vde,
 216                                        unsigned int table,
 217                                        unsigned int row,
 218                                        u32 value1, u32 value2)
 219 {
 220         u32 *iram_tables = vde->iram;
 221
 222         trace_vde_setup_iram_entry(table, row, value1, value2);
 223
 224         iram_tables[0x20 * table + row * 2] = value1;
 225         iram_tables[0x20 * table + row * 2 + 1] = value2;
 226 }
 227
 228 static void tegra_vde_setup_iram_tables(struct tegra_vde *vde,
 229                                         struct video_frame *dpb_frames,
 230                                         unsigned int ref_frames_nb,
 231                                         unsigned int with_earlier_poc_nb)
 232 {
 233         struct video_frame *frame;
 234         u32 value, aux_addr;
 235         int with_later_poc_nb;
 236         unsigned int i, k;
 237
 238         trace_vde_ref_l0(dpb_frames[0].frame_num);
 239
 240         for (i = 0; i < 16; i++) {
 241                 if (i < ref_frames_nb) {
 242                         frame = &dpb_frames[i + 1];
 243
 244                         aux_addr = frame->aux_addr;
 245
 246                         value  = (i + 1) << 26;
 247                         value |= !(frame->flags & FLAG_B_FRAME) << 25;
 248                         value |= 1 << 24;
 249                         value |= frame->frame_num;
 250                 } else {
 251                         aux_addr = 0x6ADEAD00;
 252                         value = 0;
 253                 }
 254
 255                 tegra_vde_setup_iram_entry(vde, 0, i, value, aux_addr);
 256                 tegra_vde_setup_iram_entry(vde, 1, i, value, aux_addr);
 257                 tegra_vde_setup_iram_entry(vde, 2, i, value, aux_addr);
 258                 tegra_vde_setup_iram_entry(vde, 3, i, value, aux_addr);
 259         }
 260
 261         if (!(dpb_frames[0].flags & FLAG_B_FRAME))
 262                 return;
 263
 264         if (with_earlier_poc_nb >= ref_frames_nb)
 265                 return;
 266
 267         with_later_poc_nb = ref_frames_nb - with_earlier_poc_nb;
 268
 269         trace_vde_ref_l1(with_later_poc_nb, with_earlier_poc_nb);
 270
 271         for (i = 0, k = with_earlier_poc_nb; i < with_later_poc_nb; i++, k++) {
 272                 frame = &dpb_frames[k + 1];
 273
 274                 aux_addr = frame->aux_addr;
 275
 276                 value  = (k + 1) << 26;
 277                 value |= !(frame->flags & FLAG_B_FRAME) << 25;
 278                 value |= 1 << 24;
 279                 value |= frame->frame_num;
 280
 281                 tegra_vde_setup_iram_entry(vde, 2, i, value, aux_addr);
 282         }
 283
 284         for (k = 0; i < ref_frames_nb; i++, k++) {
 285                 frame = &dpb_frames[k + 1];
 286
 287                 aux_addr = frame->aux_addr;
 288
 289                 value  = (k + 1) << 26;
 290                 value |= !(frame->flags & FLAG_B_FRAME) << 25;
 291                 value |= 1 << 24;
 292                 value |= frame->frame_num;
 293
 294                 tegra_vde_setup_iram_entry(vde, 2, i, value, aux_addr);
 295         }
 296 }
 297
 298 static int tegra_vde_setup_hw_context(struct tegra_vde *vde,
 299                                       struct tegra_vde_h264_decoder_ctx *ctx,
 300                                       struct video_frame *dpb_frames,
 301                                       dma_addr_t bitstream_data_addr,
 302                                       size_t bitstream_data_size,
 303                                       unsigned int macroblocks_nb)
 304 {
 305         struct device *dev = vde->miscdev.parent;
 306         u32 value;
 307         int err;
 308
 309         tegra_vde_set_bits(vde, 0x000A, vde->sxe, 0xF0);
 310         tegra_vde_set_bits(vde, 0x000B, vde->bsev, CMDQUE_CONTROL);
 311         tegra_vde_set_bits(vde, 0x8002, vde->mbe, 0x50);
 312         tegra_vde_set_bits(vde, 0x000A, vde->mbe, 0xA0);
 313         tegra_vde_set_bits(vde, 0x000A, vde->ppe, 0x14);
 314         tegra_vde_set_bits(vde, 0x000A, vde->ppe, 0x28);
 315         tegra_vde_set_bits(vde, 0x0A00, vde->mce, 0x08);
 316         tegra_vde_set_bits(vde, 0x000A, vde->tfe, 0x00);
 317         tegra_vde_set_bits(vde, 0x0005, vde->vdma, 0x04);
 318
 319         tegra_vde_writel(vde, 0x00000000, vde->vdma, 0x1C);
 320         tegra_vde_writel(vde, 0x00000000, vde->vdma, 0x00);
 321         tegra_vde_writel(vde, 0x00000007, vde->vdma, 0x04);
 322         tegra_vde_writel(vde, 0x00000007, vde->frameid, 0x200);
 323         tegra_vde_writel(vde, 0x00000005, vde->tfe, 0x04);
 324         tegra_vde_writel(vde, 0x00000000, vde->mbe, 0x84);
 325         tegra_vde_writel(vde, 0x00000010, vde->sxe, 0x08);
 326         tegra_vde_writel(vde, 0x00000150, vde->sxe, 0x54);
 327         tegra_vde_writel(vde, 0x0000054C, vde->sxe, 0x58);
 328         tegra_vde_writel(vde, 0x00000E34, vde->sxe, 0x5C);
 329         tegra_vde_writel(vde, 0x063C063C, vde->mce, 0x10);
 330         tegra_vde_writel(vde, 0x0003FC00, vde->bsev, INTR_STATUS);
 331         tegra_vde_writel(vde, 0x0000150D, vde->bsev, BSE_CONFIG);
 332         tegra_vde_writel(vde, 0x00000100, vde->bsev, BSE_INT_ENB);
 333         tegra_vde_writel(vde, 0x00000000, vde->bsev, 0x98);
 334         tegra_vde_writel(vde, 0x00000060, vde->bsev, 0x9C);
 335
 336         memset(vde->iram + 128, 0, macroblocks_nb / 2);
 337
 338         tegra_setup_frameidx(vde, dpb_frames, ctx->dpb_frames_nb,
 339                              ctx->pic_width_in_mbs, ctx->pic_height_in_mbs);
 340
 341         tegra_vde_setup_iram_tables(vde, dpb_frames,
 342                                     ctx->dpb_frames_nb - 1,
 343                                     ctx->dpb_ref_frames_with_earlier_poc_nb);
 344
 345         /*
 346          * The IRAM mapping is write-combine, ensure that CPU buffers have
 347          * been flushed at this point.
 348          */
 349         wmb();
 350
 351         tegra_vde_writel(vde, 0x00000000, vde->bsev, 0x8C);
 352         tegra_vde_writel(vde, bitstream_data_addr + bitstream_data_size,
 353                          vde->bsev, 0x54);
 354
 355         value = ctx->pic_width_in_mbs << 11 | ctx->pic_height_in_mbs << 3;
 356
 357         tegra_vde_writel(vde, value, vde->bsev, 0x88);
 358
 359         err = tegra_vde_wait_bsev(vde, false);
 360         if (err)
 361                 return err;
 362
 363         err = tegra_vde_push_to_bsev_icmdqueue(vde, 0x800003FC, false);
 364         if (err)
 365                 return err;
 366
 367         value = 0x01500000;
 368         value |= ((vde->iram_lists_addr + 512) >> 2) & 0xFFFF;
 369
 370         err = tegra_vde_push_to_bsev_icmdqueue(vde, value, true);
 371         if (err)
 372                 return err;
 373
 374         err = tegra_vde_push_to_bsev_icmdqueue(vde, 0x840F054C, false);
 375         if (err)
 376                 return err;
 377
 378         err = tegra_vde_push_to_bsev_icmdqueue(vde, 0x80000080, false);
 379         if (err)
 380                 return err;
 381
 382         value = 0x0E340000 | ((vde->iram_lists_addr >> 2) & 0xFFFF);
 383
 384         err = tegra_vde_push_to_bsev_icmdqueue(vde, value, true);
 385         if (err)
 386                 return err;
 387
 388         value = 0x00800005;
 389         value |= ctx->pic_width_in_mbs << 11;
 390         value |= ctx->pic_height_in_mbs << 3;
 391
 392         tegra_vde_writel(vde, value, vde->sxe, 0x10);
 393
 394         value = !ctx->baseline_profile << 17;
 395         value |= ctx->level_idc << 13;
 396         value |= ctx->log2_max_pic_order_cnt_lsb << 7;
 397         value |= ctx->pic_order_cnt_type << 5;
 398         value |= ctx->log2_max_frame_num;
 399
 400         tegra_vde_writel(vde, value, vde->sxe, 0x40);
 401
 402         value = ctx->pic_init_qp << 25;
 403         value |= !!(ctx->deblocking_filter_control_present_flag) << 2;
 404         value |= !!ctx->pic_order_present_flag;
 405
 406         tegra_vde_writel(vde, value, vde->sxe, 0x44);
 407
 408         value = ctx->chroma_qp_index_offset;
 409         value |= ctx->num_ref_idx_l0_active_minus1 << 5;
 410         value |= ctx->num_ref_idx_l1_active_minus1 << 10;
 411         value |= !!ctx->constrained_intra_pred_flag << 15;
 412
 413         tegra_vde_writel(vde, value, vde->sxe, 0x48);
 414
 415         value = 0x0C000000;
 416         value |= !!(dpb_frames[0].flags & FLAG_B_FRAME) << 24;
 417
 418         tegra_vde_writel(vde, value, vde->sxe, 0x4C);
 419
 420         value = 0x03800000;
 421         value |= bitstream_data_size & GENMASK(19, 15);
 422
 423         tegra_vde_writel(vde, value, vde->sxe, 0x68);
 424
 425         tegra_vde_writel(vde, bitstream_data_addr, vde->sxe, 0x6C);
 426
 427         value = 0x10000005;
 428         value |= ctx->pic_width_in_mbs << 11;
 429         value |= ctx->pic_height_in_mbs << 3;
 430
 431         tegra_vde_writel(vde, value, vde->mbe, 0x80);
 432
 433         value = 0x26800000;
 434         value |= ctx->level_idc << 4;
 435         value |= !ctx->baseline_profile << 1;
 436         value |= !!ctx->direct_8x8_inference_flag;
 437
 438         tegra_vde_writel(vde, value, vde->mbe, 0x80);
 439
 440         tegra_vde_writel(vde, 0xF4000001, vde->mbe, 0x80);
 441         tegra_vde_writel(vde, 0x20000000, vde->mbe, 0x80);
 442         tegra_vde_writel(vde, 0xF4000101, vde->mbe, 0x80);
 443
 444         value = 0x20000000;
 445         value |= ctx->chroma_qp_index_offset << 8;
 446
 447         tegra_vde_writel(vde, value, vde->mbe, 0x80);
 448
 449         err = tegra_vde_setup_mbe_frame_idx(vde,
 450                                             ctx->dpb_frames_nb - 1,
 451                                             ctx->pic_order_cnt_type == 0);
 452         if (err) {
 453                 dev_err(dev, "MBE frames setup failed %d\n", err);
 454                 return err;
 455         }
 456
 457         tegra_vde_mbe_set_0xa_reg(vde, 0, 0x000009FC);
 458         tegra_vde_mbe_set_0xa_reg(vde, 2, 0x61DEAD00);
 459         tegra_vde_mbe_set_0xa_reg(vde, 4, 0x62DEAD00);
 460         tegra_vde_mbe_set_0xa_reg(vde, 6, 0x63DEAD00);
 461         tegra_vde_mbe_set_0xa_reg(vde, 8, dpb_frames[0].aux_addr);
 462
 463         value = 0xFC000000;
 464         value |= !!(dpb_frames[0].flags & FLAG_B_FRAME) << 2;
 465
 466         if (!ctx->baseline_profile)
 467                 value |= !!(dpb_frames[0].flags & FLAG_REFERENCE) << 1;
 468
 469         tegra_vde_writel(vde, value, vde->mbe, 0x80);
 470
 471         err = tegra_vde_wait_mbe(vde);
 472         if (err) {
 473                 dev_err(dev, "MBE programming failed %d\n", err);
 474                 return err;
 475         }
 476
 477         return 0;
 478 }
 479
 480 static void tegra_vde_decode_frame(struct tegra_vde *vde,
 481                                    unsigned int macroblocks_nb)
 482 {
 483         reinit_completion(&vde->decode_completion);
 484
 485         tegra_vde_writel(vde, 0x00000001, vde->bsev, 0x8C);
 486         tegra_vde_writel(vde, 0x20000000 | (macroblocks_nb - 1),
 487                          vde->sxe, 0x00);
 488 }
 489
 490 static int tegra_vde_attach_dmabuf(struct tegra_vde *vde,
 491                                    int fd,
 492                                    unsigned long offset,
 493                                    size_t min_size,
 494                                    size_t align_size,
 495                                    struct dma_buf_attachment **a,
 496                                    dma_addr_t *addrp,
 497                                    size_t *size,
 498                                    enum dma_data_direction dma_dir)
 499 {
 500         struct device *dev = vde->miscdev.parent;
 501         struct dma_buf *dmabuf;
 502         int err;
 503
 504         dmabuf = dma_buf_get(fd);
 505         if (IS_ERR(dmabuf)) {
 506                 dev_err(dev, "Invalid dmabuf FD\n");
 507                 return PTR_ERR(dmabuf);
 508         }
 509
 510         if (dmabuf->size & (align_size - 1)) {
 511                 dev_err(dev, "Unaligned dmabuf 0x%zX, should be aligned to 0x%zX\n",
 512                         dmabuf->size, align_size);
 513                 return -EINVAL;
 514         }
 515
 516         if ((u64)offset + min_size > dmabuf->size) {
 517                 dev_err(dev, "Too small dmabuf size %zu @0x%lX, should be at least %zu\n",
 518                         dmabuf->size, offset, min_size);
 519                 return -EINVAL;
 520         }
 521
 522         err = tegra_vde_dmabuf_cache_map(vde, dmabuf, dma_dir, a, addrp);
 523         if (err)
 524                 goto err_put;
 525
 526         *addrp = *addrp + offset;
 527
 528         if (size)
 529                 *size = dmabuf->size - offset;
 530
 531         return 0;
 532
 533 err_put:
 534         dma_buf_put(dmabuf);
 535
 536         return err;
 537 }
 538
 539 static int tegra_vde_attach_dmabufs_to_frame(struct tegra_vde *vde,
 540                                              struct video_frame *frame,
 541                                              struct tegra_vde_h264_frame *src,
 542                                              enum dma_data_direction dma_dir,
 543                                              bool baseline_profile,
 544                                              size_t lsize, size_t csize)
 545 {
 546         int err;
 547
 548         err = tegra_vde_attach_dmabuf(vde, src->y_fd,
 549                                       src->y_offset, lsize, SZ_256,
 550                                       &frame->y_dmabuf_attachment,
 551                                       &frame->y_addr,
 552                                       NULL, dma_dir);
 553         if (err)
 554                 return err;
 555
 556         err = tegra_vde_attach_dmabuf(vde, src->cb_fd,
 557                                       src->cb_offset, csize, SZ_256,
 558                                       &frame->cb_dmabuf_attachment,
 559                                       &frame->cb_addr,
 560                                       NULL, dma_dir);
 561         if (err)
 562                 goto err_release_y;
 563
 564         err = tegra_vde_attach_dmabuf(vde, src->cr_fd,
 565                                       src->cr_offset, csize, SZ_256,
 566                                       &frame->cr_dmabuf_attachment,
 567                                       &frame->cr_addr,
 568                                       NULL, dma_dir);
 569         if (err)
 570                 goto err_release_cb;
 571
 572         if (baseline_profile) {
 573                 frame->aux_addr = 0x64DEAD00;
 574                 return 0;
 575         }
 576
 577         err = tegra_vde_attach_dmabuf(vde, src->aux_fd,
 578                                       src->aux_offset, csize, SZ_256,
 579                                       &frame->aux_dmabuf_attachment,
 580                                       &frame->aux_addr,
 581                                       NULL, dma_dir);
 582         if (err)
 583                 goto err_release_cr;
 584
 585         return 0;
 586
 587 err_release_cr:
 588         tegra_vde_dmabuf_cache_unmap(vde, frame->cr_dmabuf_attachment, true);
 589 err_release_cb:
 590         tegra_vde_dmabuf_cache_unmap(vde, frame->cb_dmabuf_attachment, true);
 591 err_release_y:
 592         tegra_vde_dmabuf_cache_unmap(vde, frame->y_dmabuf_attachment, true);
 593
 594         return err;
 595 }
 596
 597 static void tegra_vde_release_frame_dmabufs(struct tegra_vde *vde,
 598                                             struct video_frame *frame,
 599                                             enum dma_data_direction dma_dir,
 600                                             bool baseline_profile,
 601                                             bool release)
 602 {
 603         if (!baseline_profile)
 604                 tegra_vde_dmabuf_cache_unmap(vde, frame->aux_dmabuf_attachment,
 605                                              release);
 606
 607         tegra_vde_dmabuf_cache_unmap(vde, frame->cr_dmabuf_attachment, release);
 608         tegra_vde_dmabuf_cache_unmap(vde, frame->cb_dmabuf_attachment, release);
 609         tegra_vde_dmabuf_cache_unmap(vde, frame->y_dmabuf_attachment, release);
 610 }
 611
 612 static int tegra_vde_validate_frame(struct device *dev,
 613                                     struct tegra_vde_h264_frame *frame)
 614 {
 615         if (frame->frame_num > 0x7FFFFF) {
 616                 dev_err(dev, "Bad frame_num %u\n", frame->frame_num);
 617                 return -EINVAL;
 618         }
 619
 620         return 0;
 621 }
 622
 623 static int tegra_vde_validate_h264_ctx(struct device *dev,
 624                                        struct tegra_vde_h264_decoder_ctx *ctx)
 625 {
 626         if (ctx->dpb_frames_nb == 0 || ctx->dpb_frames_nb > 17) {
 627                 dev_err(dev, "Bad DPB size %u\n", ctx->dpb_frames_nb);
 628                 return -EINVAL;
 629         }
 630
 631         if (ctx->level_idc > 15) {
 632                 dev_err(dev, "Bad level value %u\n", ctx->level_idc);
 633                 return -EINVAL;
 634         }
 635
 636         if (ctx->pic_init_qp > 52) {
 637                 dev_err(dev, "Bad pic_init_qp value %u\n", ctx->pic_init_qp);
 638                 return -EINVAL;
 639         }
 640
 641         if (ctx->log2_max_pic_order_cnt_lsb > 16) {
 642                 dev_err(dev, "Bad log2_max_pic_order_cnt_lsb value %u\n",
 643                         ctx->log2_max_pic_order_cnt_lsb);
 644                 return -EINVAL;
 645         }
 646
 647         if (ctx->log2_max_frame_num > 16) {
 648                 dev_err(dev, "Bad log2_max_frame_num value %u\n",
 649                         ctx->log2_max_frame_num);
 650                 return -EINVAL;
 651         }
 652
 653         if (ctx->chroma_qp_index_offset > 31) {
 654                 dev_err(dev, "Bad chroma_qp_index_offset value %u\n",
 655                         ctx->chroma_qp_index_offset);
 656                 return -EINVAL;
 657         }
 658
 659         if (ctx->pic_order_cnt_type > 2) {
 660                 dev_err(dev, "Bad pic_order_cnt_type value %u\n",
 661                         ctx->pic_order_cnt_type);
 662                 return -EINVAL;
 663         }
 664
 665         if (ctx->num_ref_idx_l0_active_minus1 > 15) {
 666                 dev_err(dev, "Bad num_ref_idx_l0_active_minus1 value %u\n",
 667                         ctx->num_ref_idx_l0_active_minus1);
 668                 return -EINVAL;
 669         }
 670
 671         if (ctx->num_ref_idx_l1_active_minus1 > 15) {
 672                 dev_err(dev, "Bad num_ref_idx_l1_active_minus1 value %u\n",
 673                         ctx->num_ref_idx_l1_active_minus1);
 674                 return -EINVAL;
 675         }
 676
 677         if (!ctx->pic_width_in_mbs || ctx->pic_width_in_mbs > 127) {
 678                 dev_err(dev, "Bad pic_width_in_mbs value %u\n",
 679                         ctx->pic_width_in_mbs);
 680                 return -EINVAL;
 681         }
 682
 683         if (!ctx->pic_height_in_mbs || ctx->pic_height_in_mbs > 127) {
 684                 dev_err(dev, "Bad pic_height_in_mbs value %u\n",
 685                         ctx->pic_height_in_mbs);
 686                 return -EINVAL;
 687         }
 688
 689         return 0;
 690 }
 691
 692 static int tegra_vde_ioctl_decode_h264(struct tegra_vde *vde,
 693                                        unsigned long vaddr)
 694 {
 695         struct device *dev = vde->miscdev.parent;
 696         struct tegra_vde_h264_decoder_ctx ctx;
 697         struct tegra_vde_h264_frame *frames;
 698         struct tegra_vde_h264_frame __user *frames_user;
 699         struct video_frame *dpb_frames;
 700         struct dma_buf_attachment *bitstream_data_dmabuf_attachment;
 701         enum dma_data_direction dma_dir;
 702         dma_addr_t bitstream_data_addr;
 703         dma_addr_t bsev_ptr;
 704         size_t lsize, csize;
 705         size_t bitstream_data_size;
 706         unsigned int macroblocks_nb;
 707         unsigned int read_bytes;
 708         unsigned int cstride;
 709         unsigned int i;
 710         long timeout;
 711         int ret, err;
 712
 713         if (copy_from_user(&ctx, (void __user *)vaddr, sizeof(ctx)))
 714                 return -EFAULT;
 715
 716         ret = tegra_vde_validate_h264_ctx(dev, &ctx);
 717         if (ret)
 718                 return ret;
 719
 720         ret = tegra_vde_attach_dmabuf(vde, ctx.bitstream_data_fd,
 721                                       ctx.bitstream_data_offset,
 722                                       SZ_16K, SZ_16K,
 723                                       &bitstream_data_dmabuf_attachment,
 724                                       &bitstream_data_addr,
 725                                       &bitstream_data_size,
 726                                       DMA_TO_DEVICE);
 727         if (ret)
 728                 return ret;
 729
 730         frames = kmalloc_array(ctx.dpb_frames_nb, sizeof(*frames), GFP_KERNEL);
 731         if (!frames) {
 732                 ret = -ENOMEM;
 733                 goto release_bitstream_dmabuf;
 734         }
 735
 736         dpb_frames = kcalloc(ctx.dpb_frames_nb, sizeof(*dpb_frames),
 737                              GFP_KERNEL);
 738         if (!dpb_frames) {
 739                 ret = -ENOMEM;
 740                 goto free_frames;
 741         }
 742
 743         macroblocks_nb = ctx.pic_width_in_mbs * ctx.pic_height_in_mbs;
 744         frames_user = u64_to_user_ptr(ctx.dpb_frames_ptr);
 745
 746         if (copy_from_user(frames, frames_user,
 747                            ctx.dpb_frames_nb * sizeof(*frames))) {
 748                 ret = -EFAULT;
 749                 goto free_dpb_frames;
 750         }
 751
 752         cstride = ALIGN(ctx.pic_width_in_mbs * 8, 16);
 753         csize = cstride * ctx.pic_height_in_mbs * 8;
 754         lsize = macroblocks_nb * 256;
 755
 756         for (i = 0; i < ctx.dpb_frames_nb; i++) {
 757                 ret = tegra_vde_validate_frame(dev, &frames[i]);
 758                 if (ret)
 759                         goto release_dpb_frames;
 760
 761                 dpb_frames[i].flags = frames[i].flags;
 762                 dpb_frames[i].frame_num = frames[i].frame_num;
 763
 764                 dma_dir = (i == 0) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 765
 766                 ret = tegra_vde_attach_dmabufs_to_frame(vde, &dpb_frames[i],
 767                                                         &frames[i], dma_dir,
 768                                                         ctx.baseline_profile,
 769                                                         lsize, csize);
 770                 if (ret)
 771                         goto release_dpb_frames;
 772         }
 773
 774         ret = mutex_lock_interruptible(&vde->lock);
 775         if (ret)
 776                 goto release_dpb_frames;
 777
 778         ret = pm_runtime_get_sync(dev);
 779         if (ret < 0)
 780                 goto put_runtime_pm;
 781
 782         /*
 783          * We rely on the VDE registers reset value, otherwise VDE
 784          * causes bus lockup.
 785          */
 786         ret = reset_control_assert(vde->rst_mc);
 787         if (ret) {
 788                 dev_err(dev, "DEC start: Failed to assert MC reset: %d\n",
 789                         ret);
 790                 goto put_runtime_pm;
 791         }
 792
 793         ret = reset_control_reset(vde->rst);
 794         if (ret) {
 795                 dev_err(dev, "DEC start: Failed to reset HW: %d\n", ret);
 796                 goto put_runtime_pm;
 797         }
 798
 799         ret = reset_control_deassert(vde->rst_mc);
 800         if (ret) {
 801                 dev_err(dev, "DEC start: Failed to deassert MC reset: %d\n",
 802                         ret);
 803                 goto put_runtime_pm;
 804         }
 805
 806         ret = tegra_vde_setup_hw_context(vde, &ctx, dpb_frames,
 807                                          bitstream_data_addr,
 808                                          bitstream_data_size,
 809                                          macroblocks_nb);
 810         if (ret)
 811                 goto put_runtime_pm;
 812
 813         tegra_vde_decode_frame(vde, macroblocks_nb);
 814
 815         timeout = wait_for_completion_interruptible_timeout(
 816                         &vde->decode_completion, msecs_to_jiffies(1000));
 817         if (timeout == 0) {
 818                 bsev_ptr = tegra_vde_readl(vde, vde->bsev, 0x10);
 819                 macroblocks_nb = tegra_vde_readl(vde, vde->sxe, 0xC8) & 0x1FFF;
 820                 read_bytes = bsev_ptr ? bsev_ptr - bitstream_data_addr : 0;
 821
 822                 dev_err(dev, "Decoding failed: read 0x%X bytes, %u macroblocks parsed\n",
 823                         read_bytes, macroblocks_nb);
 824
 825                 ret = -EIO;
 826         } else if (timeout < 0) {
 827                 ret = timeout;
 828         }
 829
 830         /*
 831          * At first reset memory client to avoid resetting VDE HW in the
 832          * middle of DMA which could result into memory corruption or hang
 833          * the whole system.
 834          */
 835         err = reset_control_assert(vde->rst_mc);
 836         if (err)
 837                 dev_err(dev, "DEC end: Failed to assert MC reset: %d\n", err);
 838
 839         err = reset_control_assert(vde->rst);
 840         if (err)
 841                 dev_err(dev, "DEC end: Failed to assert HW reset: %d\n", err);
 842
 843 put_runtime_pm:
 844         pm_runtime_mark_last_busy(dev);
 845         pm_runtime_put_autosuspend(dev);
 846         mutex_unlock(&vde->lock);
 847
 848 release_dpb_frames:
 849         while (i--) {
 850                 dma_dir = (i == 0) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 851
 852                 tegra_vde_release_frame_dmabufs(vde, &dpb_frames[i], dma_dir,
 853                                                 ctx.baseline_profile, ret != 0);
 854         }
 855
 856 free_dpb_frames:
 857         kfree(dpb_frames);
 858
 859 free_frames:
 860         kfree(frames);
 861
 862 release_bitstream_dmabuf:
 863         tegra_vde_dmabuf_cache_unmap(vde, bitstream_data_dmabuf_attachment,
 864                                      ret != 0);
 865
 866         return ret;
 867 }
 868
 869 static long tegra_vde_unlocked_ioctl(struct file *filp,
 870                                      unsigned int cmd, unsigned long arg)
 871 {
 872         struct miscdevice *miscdev = filp->private_data;
 873         struct tegra_vde *vde = container_of(miscdev, struct tegra_vde,
 874                                              miscdev);
 875
 876         switch (cmd) {
 877         case TEGRA_VDE_IOCTL_DECODE_H264:
 878                 return tegra_vde_ioctl_decode_h264(vde, arg);
 879         }
 880
 881         dev_err(miscdev->parent, "Invalid IOCTL command %u\n", cmd);
 882
 883         return -ENOTTY;
 884 }
 885
 886 static int tegra_vde_release_file(struct inode *inode, struct file *filp)
 887 {
 888         struct miscdevice *miscdev = filp->private_data;
 889         struct tegra_vde *vde = container_of(miscdev, struct tegra_vde,
 890                                              miscdev);
 891
 892         tegra_vde_dmabuf_cache_unmap_sync(vde);
 893
 894         return 0;
 895 }
 896
 897 static const struct file_operations tegra_vde_fops = {
 898         .owner          = THIS_MODULE,
 899         .unlocked_ioctl = tegra_vde_unlocked_ioctl,
 900         .release        = tegra_vde_release_file,
 901 };
 902
 903 static irqreturn_t tegra_vde_isr(int irq, void *data)
 904 {
 905         struct tegra_vde *vde = data;
 906
 907         if (completion_done(&vde->decode_completion))
 908                 return IRQ_NONE;
 909
 910         tegra_vde_set_bits(vde, 0, vde->frameid, 0x208);
 911         complete(&vde->decode_completion);
 912
 913         return IRQ_HANDLED;
 914 }
 915
 916 static __maybe_unused int tegra_vde_runtime_suspend(struct device *dev)
 917 {
 918         struct tegra_vde *vde = dev_get_drvdata(dev);
 919         int err;
 920
 921         err = tegra_powergate_power_off(TEGRA_POWERGATE_VDEC);
 922         if (err) {
 923                 dev_err(dev, "Failed to power down HW: %d\n", err);
 924                 return err;
 925         }
 926
 927         clk_disable_unprepare(vde->clk);
 928
 929         return 0;
 930 }
 931
 932 static __maybe_unused int tegra_vde_runtime_resume(struct device *dev)
 933 {
 934         struct tegra_vde *vde = dev_get_drvdata(dev);
 935         int err;
 936
 937         err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_VDEC,
 938                                                 vde->clk, vde->rst);
 939         if (err) {
 940                 dev_err(dev, "Failed to power up HW : %d\n", err);
 941                 return err;
 942         }
 943
 944         return 0;
 945 }
 946
 947 static int tegra_vde_probe(struct platform_device *pdev)
 948 {
 949         struct device *dev = &pdev->dev;
 950         struct tegra_vde *vde;
 951         int irq, err;
 952
 953         vde = devm_kzalloc(dev, sizeof(*vde), GFP_KERNEL);
 954         if (!vde)
 955                 return -ENOMEM;
 956
 957         platform_set_drvdata(pdev, vde);
 958
 959         vde->sxe = devm_platform_ioremap_resource_byname(pdev, "sxe");
 960         if (IS_ERR(vde->sxe))
 961                 return PTR_ERR(vde->sxe);
 962
 963         vde->bsev = devm_platform_ioremap_resource_byname(pdev, "bsev");
 964         if (IS_ERR(vde->bsev))
 965                 return PTR_ERR(vde->bsev);
 966
 967         vde->mbe = devm_platform_ioremap_resource_byname(pdev, "mbe");
 968         if (IS_ERR(vde->mbe))
 969                 return PTR_ERR(vde->mbe);
 970
 971         vde->ppe = devm_platform_ioremap_resource_byname(pdev, "ppe");
 972         if (IS_ERR(vde->ppe))
 973                 return PTR_ERR(vde->ppe);
 974
 975         vde->mce = devm_platform_ioremap_resource_byname(pdev, "mce");
 976         if (IS_ERR(vde->mce))
 977                 return PTR_ERR(vde->mce);
 978
 979         vde->tfe = devm_platform_ioremap_resource_byname(pdev, "tfe");
 980         if (IS_ERR(vde->tfe))
 981                 return PTR_ERR(vde->tfe);
 982
 983         vde->ppb = devm_platform_ioremap_resource_byname(pdev, "ppb");
 984         if (IS_ERR(vde->ppb))
 985                 return PTR_ERR(vde->ppb);
 986
 987         vde->vdma = devm_platform_ioremap_resource_byname(pdev, "vdma");
 988         if (IS_ERR(vde->vdma))
 989                 return PTR_ERR(vde->vdma);
 990
 991         vde->frameid = devm_platform_ioremap_resource_byname(pdev, "frameid");
 992         if (IS_ERR(vde->frameid))
 993                 return PTR_ERR(vde->frameid);
 994
 995         vde->clk = devm_clk_get(dev, NULL);
 996         if (IS_ERR(vde->clk)) {
 997                 err = PTR_ERR(vde->clk);
 998                 dev_err(dev, "Could not get VDE clk %d\n", err);
 999                 return err;
1000         }
1001
1002         vde->rst = devm_reset_control_get(dev, NULL);
1003         if (IS_ERR(vde->rst)) {
1004                 err = PTR_ERR(vde->rst);
1005                 dev_err(dev, "Could not get VDE reset %d\n", err);
1006                 return err;
1007         }
1008
1009         vde->rst_mc = devm_reset_control_get_optional(dev, "mc");
1010         if (IS_ERR(vde->rst_mc)) {
1011                 err = PTR_ERR(vde->rst_mc);
1012                 dev_err(dev, "Could not get MC reset %d\n", err);
1013                 return err;
1014         }
1015
1016         irq = platform_get_irq_byname(pdev, "sync-token");
1017         if (irq < 0)
1018                 return irq;
1019
1020         err = devm_request_irq(dev, irq, tegra_vde_isr, 0,
1021                                dev_name(dev), vde);
1022         if (err) {
1023                 dev_err(dev, "Could not request IRQ %d\n", err);
1024                 return err;
1025         }
1026
1027         vde->iram_pool = of_gen_pool_get(dev->of_node, "iram", 0);
1028         if (!vde->iram_pool) {
1029                 dev_err(dev, "Could not get IRAM pool\n");
1030                 return -EPROBE_DEFER;
1031         }
1032
1033         vde->iram = gen_pool_dma_alloc(vde->iram_pool,
1034                                        gen_pool_size(vde->iram_pool),
1035                                        &vde->iram_lists_addr);
1036         if (!vde->iram) {
1037                 dev_err(dev, "Could not reserve IRAM\n");
1038                 return -ENOMEM;
1039         }
1040
1041         INIT_LIST_HEAD(&vde->map_list);
1042         mutex_init(&vde->map_lock);
1043         mutex_init(&vde->lock);
1044         init_completion(&vde->decode_completion);
1045
1046         vde->miscdev.minor = MISC_DYNAMIC_MINOR;
1047         vde->miscdev.name = "tegra_vde";
1048         vde->miscdev.fops = &tegra_vde_fops;
1049         vde->miscdev.parent = dev;
1050
1051         err = tegra_vde_iommu_init(vde);
1052         if (err) {
1053                 dev_err(dev, "Failed to initialize IOMMU: %d\n", err);
1054                 goto err_gen_free;
1055         }
1056
1057         err = misc_register(&vde->miscdev);
1058         if (err) {
1059                 dev_err(dev, "Failed to register misc device: %d\n", err);
1060                 goto err_deinit_iommu;
1061         }
1062
1063         pm_runtime_enable(dev);
1064         pm_runtime_use_autosuspend(dev);
1065         pm_runtime_set_autosuspend_delay(dev, 300);
1066
1067         /*
1068          * VDE partition may be left ON after bootloader, hence let's
1069          * power-cycle it in order to put hardware into a predictable lower
1070          * power state.
1071          */
1072         pm_runtime_get_sync(dev);
1073         pm_runtime_put(dev);
1074
1075         return 0;
1076
1077 err_deinit_iommu:
1078         tegra_vde_iommu_deinit(vde);
1079
1080 err_gen_free:
1081         gen_pool_free(vde->iram_pool, (unsigned long)vde->iram,
1082                       gen_pool_size(vde->iram_pool));
1083
1084         return err;
1085 }
1086
1087 static int tegra_vde_remove(struct platform_device *pdev)
1088 {
1089         struct tegra_vde *vde = platform_get_drvdata(pdev);
1090         struct device *dev = &pdev->dev;
1091
1092         pm_runtime_get_sync(dev);
1093         pm_runtime_dont_use_autosuspend(dev);
1094         pm_runtime_disable(dev);
1095
1096         /*
1097          * Balance RPM state, the VDE power domain is left ON and hardware
1098          * is clock-gated. It's safe to reboot machine now.
1099          */
1100         pm_runtime_put_noidle(dev);
1101         clk_disable_unprepare(vde->clk);
1102
1103         misc_deregister(&vde->miscdev);
1104
1105         tegra_vde_dmabuf_cache_unmap_all(vde);
1106         tegra_vde_iommu_deinit(vde);
1107
1108         gen_pool_free(vde->iram_pool, (unsigned long)vde->iram,
1109                       gen_pool_size(vde->iram_pool));
1110
1111         return 0;
1112 }
1113
1114 static void tegra_vde_shutdown(struct platform_device *pdev)
1115 {
1116         /*
1117          * On some devices bootloader isn't ready to a power-gated VDE on
1118          * a warm-reboot, machine will hang in that case.
1119          */
1120         if (pm_runtime_status_suspended(&pdev->dev))
1121                 tegra_vde_runtime_resume(&pdev->dev);
1122 }
1123
1124 static __maybe_unused int tegra_vde_pm_suspend(struct device *dev)
1125 {
1126         struct tegra_vde *vde = dev_get_drvdata(dev);
1127         int err;
1128
1129         mutex_lock(&vde->lock);
1130
1131         err = pm_runtime_force_suspend(dev);
1132         if (err < 0)
1133                 return err;
1134
1135         return 0;
1136 }
1137
1138 static __maybe_unused int tegra_vde_pm_resume(struct device *dev)
1139 {
1140         struct tegra_vde *vde = dev_get_drvdata(dev);
1141         int err;
1142
1143         err = pm_runtime_force_resume(dev);
1144         if (err < 0)
1145                 return err;
1146
1147         mutex_unlock(&vde->lock);
1148
1149         return 0;
1150 }
1151
1152 static const struct dev_pm_ops tegra_vde_pm_ops = {
1153         SET_RUNTIME_PM_OPS(tegra_vde_runtime_suspend,
1154                            tegra_vde_runtime_resume,
1155                            NULL)
1156         SET_SYSTEM_SLEEP_PM_OPS(tegra_vde_pm_suspend,
1157                                 tegra_vde_pm_resume)
1158 };
1159
1160 static const struct of_device_id tegra_vde_of_match[] = {
1161         { .compatible = "nvidia,tegra20-vde", },
1162         { },
1163 };
1164 MODULE_DEVICE_TABLE(of, tegra_vde_of_match);
1165
1166 static struct platform_driver tegra_vde_driver = {
1167         .probe          = tegra_vde_probe,
1168         .remove         = tegra_vde_remove,
1169         .shutdown       = tegra_vde_shutdown,
1170         .driver         = {
1171                 .name           = "tegra-vde",
1172                 .of_match_table = tegra_vde_of_match,
1173                 .pm             = &tegra_vde_pm_ops,
1174         },
1175 };
1176 module_platform_driver(tegra_vde_driver);
1177
1178 MODULE_DESCRIPTION("NVIDIA Tegra Video Decoder driver");
1179 MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>");
1180 MODULE_LICENSE("GPL");