tegra-video/tegra210.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 /*
   7  * This source file contains Tegra210 supported video formats,
   8  * VI and CSI SoC specific data, operations and registers accessors.
   9  */
  10 #include <linux/bitfield.h>
  11 #include <linux/clk.h>
  12 #include <linux/clk/tegra.h>
  13 #include <linux/delay.h>
  14 #include <linux/host1x.h>
  15 #include <linux/kthread.h>
  16
  17 #include "csi.h"
  18 #include "vi.h"
  19
  20 #define TEGRA_VI_SYNCPT_WAIT_TIMEOUT                    msecs_to_jiffies(200)
  21
  22 /* Tegra210 VI registers */
  23 #define TEGRA_VI_CFG_VI_INCR_SYNCPT                     0x000
  24 #define   VI_CFG_VI_INCR_SYNCPT_COND(x)                 (((x) & 0xff) << 8)
  25 #define   VI_CSI_PP_FRAME_START(port)                   (5 + (port) * 4)
  26 #define   VI_CSI_MW_ACK_DONE(port)                      (7 + (port) * 4)
  27 #define TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL               0x004
  28 #define   VI_INCR_SYNCPT_NO_STALL                       BIT(8)
  29 #define TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR               0x008
  30 #define TEGRA_VI_CFG_CG_CTRL                            0x0b8
  31 #define   VI_CG_2ND_LEVEL_EN                            0x1
  32
  33 /* Tegra210 VI CSI registers */
  34 #define TEGRA_VI_CSI_SW_RESET                           0x000
  35 #define TEGRA_VI_CSI_SINGLE_SHOT                        0x004
  36 #define   SINGLE_SHOT_CAPTURE                           0x1
  37 #define TEGRA_VI_CSI_IMAGE_DEF                          0x00c
  38 #define   BYPASS_PXL_TRANSFORM_OFFSET                   24
  39 #define   IMAGE_DEF_FORMAT_OFFSET                       16
  40 #define   IMAGE_DEF_DEST_MEM                            0x1
  41 #define TEGRA_VI_CSI_IMAGE_SIZE                         0x018
  42 #define   IMAGE_SIZE_HEIGHT_OFFSET                      16
  43 #define TEGRA_VI_CSI_IMAGE_SIZE_WC                      0x01c
  44 #define TEGRA_VI_CSI_IMAGE_DT                           0x020
  45 #define TEGRA_VI_CSI_SURFACE0_OFFSET_MSB                0x024
  46 #define TEGRA_VI_CSI_SURFACE0_OFFSET_LSB                0x028
  47 #define TEGRA_VI_CSI_SURFACE1_OFFSET_MSB                0x02c
  48 #define TEGRA_VI_CSI_SURFACE1_OFFSET_LSB                0x030
  49 #define TEGRA_VI_CSI_SURFACE2_OFFSET_MSB                0x034
  50 #define TEGRA_VI_CSI_SURFACE2_OFFSET_LSB                0x038
  51 #define TEGRA_VI_CSI_SURFACE0_STRIDE                    0x054
  52 #define TEGRA_VI_CSI_SURFACE1_STRIDE                    0x058
  53 #define TEGRA_VI_CSI_SURFACE2_STRIDE                    0x05c
  54 #define TEGRA_VI_CSI_SURFACE_HEIGHT0                    0x060
  55 #define TEGRA_VI_CSI_ERROR_STATUS                       0x084
  56
  57 /* Tegra210 CSI Pixel Parser registers: Starts from 0x838, offset 0x0 */
  58 #define TEGRA_CSI_INPUT_STREAM_CONTROL                  0x000
  59 #define   CSI_SKIP_PACKET_THRESHOLD_OFFSET              16
  60 #define TEGRA_CSI_PIXEL_STREAM_CONTROL0                 0x004
  61 #define   CSI_PP_PACKET_HEADER_SENT                     BIT(4)
  62 #define   CSI_PP_DATA_IDENTIFIER_ENABLE                 BIT(5)
  63 #define   CSI_PP_WORD_COUNT_SELECT_HEADER               BIT(6)
  64 #define   CSI_PP_CRC_CHECK_ENABLE                       BIT(7)
  65 #define   CSI_PP_WC_CHECK                               BIT(8)
  66 #define   CSI_PP_OUTPUT_FORMAT_STORE                    (0x3 << 16)
  67 #define   CSI_PPA_PAD_LINE_NOPAD                        (0x2 << 24)
  68 #define   CSI_PP_HEADER_EC_DISABLE                      (0x1 << 27)
  69 #define   CSI_PPA_PAD_FRAME_NOPAD                       (0x2 << 28)
  70 #define TEGRA_CSI_PIXEL_STREAM_CONTROL1                 0x008
  71 #define   CSI_PP_TOP_FIELD_FRAME_OFFSET                 0
  72 #define   CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET            4
  73 #define TEGRA_CSI_PIXEL_STREAM_GAP                      0x00c
  74 #define   PP_FRAME_MIN_GAP_OFFSET                       16
  75 #define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND               0x010
  76 #define   CSI_PP_ENABLE                                 0x1
  77 #define   CSI_PP_DISABLE                                0x2
  78 #define   CSI_PP_RST                                    0x3
  79 #define   CSI_PP_SINGLE_SHOT_ENABLE                     (0x1 << 2)
  80 #define   CSI_PP_START_MARKER_FRAME_MAX_OFFSET          12
  81 #define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME           0x014
  82 #define TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK           0x018
  83 #define TEGRA_CSI_PIXEL_PARSER_STATUS                   0x01c
  84
  85 /* Tegra210 CSI PHY registers */
  86 /* CSI_PHY_CIL_COMMAND_0 offset 0x0d0 from TEGRA_CSI_PIXEL_PARSER_0_BASE */
  87 #define TEGRA_CSI_PHY_CIL_COMMAND                       0x0d0
  88 #define   CSI_A_PHY_CIL_NOP                             0x0
  89 #define   CSI_A_PHY_CIL_ENABLE                          0x1
  90 #define   CSI_A_PHY_CIL_DISABLE                         0x2
  91 #define   CSI_A_PHY_CIL_ENABLE_MASK                     0x3
  92 #define   CSI_B_PHY_CIL_NOP                             (0x0 << 8)
  93 #define   CSI_B_PHY_CIL_ENABLE                          (0x1 << 8)
  94 #define   CSI_B_PHY_CIL_DISABLE                         (0x2 << 8)
  95 #define   CSI_B_PHY_CIL_ENABLE_MASK                     (0x3 << 8)
  96
  97 #define TEGRA_CSI_CIL_PAD_CONFIG0                       0x000
  98 #define   BRICK_CLOCK_A_4X                              (0x1 << 16)
  99 #define   BRICK_CLOCK_B_4X                              (0x2 << 16)
 100 #define TEGRA_CSI_CIL_PAD_CONFIG1                       0x004
 101 #define TEGRA_CSI_CIL_PHY_CONTROL                       0x008
 102 #define   CLK_SETTLE_MASK                               GENMASK(13, 8)
 103 #define   THS_SETTLE_MASK                               GENMASK(5, 0)
 104 #define TEGRA_CSI_CIL_INTERRUPT_MASK                    0x00c
 105 #define TEGRA_CSI_CIL_STATUS                            0x010
 106 #define TEGRA_CSI_CILX_STATUS                           0x014
 107 #define TEGRA_CSI_CIL_SW_SENSOR_RESET                   0x020
 108
 109 #define TEGRA_CSI_PATTERN_GENERATOR_CTRL                0x000
 110 #define   PG_MODE_OFFSET                                2
 111 #define   PG_ENABLE                                     0x1
 112 #define   PG_DISABLE                                    0x0
 113 #define TEGRA_CSI_PG_BLANK                              0x004
 114 #define   PG_VBLANK_OFFSET                              16
 115 #define TEGRA_CSI_PG_PHASE                              0x008
 116 #define TEGRA_CSI_PG_RED_FREQ                           0x00c
 117 #define   PG_RED_VERT_INIT_FREQ_OFFSET                  16
 118 #define   PG_RED_HOR_INIT_FREQ_OFFSET                   0
 119 #define TEGRA_CSI_PG_RED_FREQ_RATE                      0x010
 120 #define TEGRA_CSI_PG_GREEN_FREQ                         0x014
 121 #define   PG_GREEN_VERT_INIT_FREQ_OFFSET                16
 122 #define   PG_GREEN_HOR_INIT_FREQ_OFFSET                 0
 123 #define TEGRA_CSI_PG_GREEN_FREQ_RATE                    0x018
 124 #define TEGRA_CSI_PG_BLUE_FREQ                          0x01c
 125 #define   PG_BLUE_VERT_INIT_FREQ_OFFSET                 16
 126 #define   PG_BLUE_HOR_INIT_FREQ_OFFSET                  0
 127 #define TEGRA_CSI_PG_BLUE_FREQ_RATE                     0x020
 128 #define TEGRA_CSI_PG_AOHDR                              0x024
 129 #define TEGRA_CSI_CSI_SW_STATUS_RESET                   0x214
 130 #define TEGRA_CSI_CLKEN_OVERRIDE                        0x218
 131
 132 #define TEGRA210_CSI_PORT_OFFSET                        0x34
 133 #define TEGRA210_CSI_CIL_OFFSET                         0x0f4
 134 #define TEGRA210_CSI_TPG_OFFSET                         0x18c
 135
 136 #define CSI_PP_OFFSET(block)                            ((block) * 0x800)
 137 #define TEGRA210_VI_CSI_BASE(x)                         (0x100 + (x) * 0x100)
 138
 139 /* Tegra210 VI registers accessors */
 140 static void tegra_vi_write(struct tegra_vi_channel *chan, unsigned int addr,
 141                            u32 val)
 142 {
 143         writel_relaxed(val, chan->vi->iomem + addr);
 144 }
 145
 146 static u32 tegra_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
 147 {
 148         return readl_relaxed(chan->vi->iomem + addr);
 149 }
 150
 151 /* Tegra210 VI_CSI registers accessors */
 152 static void vi_csi_write(struct tegra_vi_channel *chan, u8 portno,
 153                          unsigned int addr, u32 val)
 154 {
 155         void __iomem *vi_csi_base;
 156
 157         vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 158
 159         writel_relaxed(val, vi_csi_base + addr);
 160 }
 161
 162 static u32 vi_csi_read(struct tegra_vi_channel *chan, u8 portno,
 163                        unsigned int addr)
 164 {
 165         void __iomem *vi_csi_base;
 166
 167         vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 168
 169         return readl_relaxed(vi_csi_base + addr);
 170 }
 171
 172 /*
 173  * Tegra210 VI channel capture operations
 174  */
 175 static int tegra_channel_capture_setup(struct tegra_vi_channel *chan,
 176                                        u8 portno)
 177 {
 178         u32 height = chan->format.height;
 179         u32 width = chan->format.width;
 180         u32 format = chan->fmtinfo->img_fmt;
 181         u32 data_type = chan->fmtinfo->img_dt;
 182         u32 word_count = (width * chan->fmtinfo->bit_width) / 8;
 183         u32 bypass_pixel_transform = BIT(BYPASS_PXL_TRANSFORM_OFFSET);
 184
 185         /*
 186          * VI Pixel transformation unit converts source pixels data format
 187          * into selected destination pixel format and aligns properly while
 188          * interfacing with memory packer.
 189          * This pixel transformation should be enabled for YUV and RGB
 190          * formats and should be bypassed for RAW formats as RAW formats
 191          * only support direct to memory.
 192          */
 193         if (chan->pg_mode || data_type == TEGRA_IMAGE_DT_YUV422_8 ||
 194             data_type == TEGRA_IMAGE_DT_RGB888)
 195                 bypass_pixel_transform = 0;
 196
 197         /*
 198          * For x8 source streaming, the source image is split onto two x4 ports
 199          * with left half to first x4 port and right half to second x4 port.
 200          * So, use split width and corresponding word count for each x4 port.
 201          */
 202         if (chan->numgangports > 1) {
 203                 width = width >> 1;
 204                 word_count = (width * chan->fmtinfo->bit_width) / 8;
 205         }
 206
 207         vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, 0xffffffff);
 208         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DEF,
 209                      bypass_pixel_transform |
 210                      (format << IMAGE_DEF_FORMAT_OFFSET) |
 211                      IMAGE_DEF_DEST_MEM);
 212         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DT, data_type);
 213         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE_WC, word_count);
 214         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE,
 215                      (height << IMAGE_SIZE_HEIGHT_OFFSET) | width);
 216         return 0;
 217 }
 218
 219 static void tegra_channel_vi_soft_reset(struct tegra_vi_channel *chan,
 220                                         u8 portno)
 221 {
 222         /* disable clock gating to enable continuous clock */
 223         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, 0);
 224         /*
 225          * Soft reset memory client interface, pixel format logic, sensor
 226          * control logic, and a shadow copy logic to bring VI to clean state.
 227          */
 228         vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0xf);
 229         usleep_range(100, 200);
 230         vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0x0);
 231
 232         /* enable back VI clock gating */
 233         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 234 }
 235
 236 static void tegra_channel_capture_error_recover(struct tegra_vi_channel *chan,
 237                                                 u8 portno)
 238 {
 239         struct v4l2_subdev *subdev;
 240         u32 val;
 241
 242         /*
 243          * Recover VI and CSI hardware blocks in case of missing frame start
 244          * events due to source not streaming or noisy csi inputs from the
 245          * external source or many outstanding frame start or MW_ACK_DONE
 246          * events which can cause CSI and VI hardware hang.
 247          * This helps to have a clean capture for next frame.
 248          */
 249         val = vi_csi_read(chan, portno, TEGRA_VI_CSI_ERROR_STATUS);
 250         dev_dbg(&chan->video.dev, "TEGRA_VI_CSI_ERROR_STATUS 0x%08x\n", val);
 251         vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, val);
 252
 253         val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 254         dev_dbg(&chan->video.dev,
 255                 "TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR 0x%08x\n", val);
 256         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 257
 258         /* recover VI by issuing software reset and re-setup for capture */
 259         tegra_channel_vi_soft_reset(chan, portno);
 260         tegra_channel_capture_setup(chan, portno);
 261
 262         /* recover CSI block */
 263         subdev = tegra_channel_get_remote_csi_subdev(chan);
 264         tegra_csi_error_recover(subdev);
 265 }
 266
 267 static struct tegra_channel_buffer *
 268 dequeue_buf_done(struct tegra_vi_channel *chan)
 269 {
 270         struct tegra_channel_buffer *buf = NULL;
 271
 272         spin_lock(&chan->done_lock);
 273         if (list_empty(&chan->done)) {
 274                 spin_unlock(&chan->done_lock);
 275                 return NULL;
 276         }
 277
 278         buf = list_first_entry(&chan->done,
 279                                struct tegra_channel_buffer, queue);
 280         if (buf)
 281                 list_del_init(&buf->queue);
 282         spin_unlock(&chan->done_lock);
 283
 284         return buf;
 285 }
 286
 287 static void release_buffer(struct tegra_vi_channel *chan,
 288                            struct tegra_channel_buffer *buf,
 289                            enum vb2_buffer_state state)
 290 {
 291         struct vb2_v4l2_buffer *vb = &buf->buf;
 292
 293         vb->sequence = chan->sequence++;
 294         vb->field = V4L2_FIELD_NONE;
 295         vb->vb2_buf.timestamp = ktime_get_ns();
 296         vb2_buffer_done(&vb->vb2_buf, state);
 297 }
 298
 299 static void tegra_channel_vi_buffer_setup(struct tegra_vi_channel *chan,
 300                                           u8 portno, u32 buf_offset,
 301                                           struct tegra_channel_buffer *buf)
 302 {
 303         int bytesperline = chan->format.bytesperline;
 304         u32 sizeimage = chan->format.sizeimage;
 305
 306         /* program buffer address by using surface 0 */
 307         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_MSB,
 308                      ((u64)buf->addr + buf_offset) >> 32);
 309         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_LSB,
 310                      buf->addr + buf_offset);
 311         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_STRIDE, bytesperline);
 312
 313         if (chan->fmtinfo->fourcc != V4L2_PIX_FMT_NV16)
 314                 return;
 315         /*
 316          * Program surface 1 for UV plane with offset sizeimage from Y plane.
 317          */
 318         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_MSB,
 319                      (((u64)buf->addr + sizeimage / 2) + buf_offset) >> 32);
 320         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_LSB,
 321                      buf->addr + sizeimage / 2 + buf_offset);
 322         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_STRIDE, bytesperline);
 323 }
 324
 325 static int tegra_channel_capture_frame(struct tegra_vi_channel *chan,
 326                                        struct tegra_channel_buffer *buf)
 327 {
 328         u32 thresh, value, frame_start, mw_ack_done;
 329         u32 fs_thresh[GANG_PORTS_MAX];
 330         u8 *portnos = chan->portnos;
 331         int gang_bpl = (chan->format.width >> 1) * chan->fmtinfo->bpp;
 332         u32 buf_offset;
 333         bool capture_timedout = false;
 334         int err, i;
 335
 336         for (i = 0; i < chan->numgangports; i++) {
 337                 /*
 338                  * Align buffers side-by-side for all consecutive x4 ports
 339                  * in gang ports using bytes per line based on source split
 340                  * width.
 341                  */
 342                 buf_offset = i * roundup(gang_bpl, SURFACE_ALIGN_BYTES);
 343                 tegra_channel_vi_buffer_setup(chan, portnos[i], buf_offset,
 344                                               buf);
 345
 346                 /*
 347                  * Tegra VI block interacts with host1x syncpt to synchronize
 348                  * programmed condition and hardware operation for capture.
 349                  * Frame start and Memory write acknowledge syncpts has their
 350                  * own FIFO of depth 2.
 351                  *
 352                  * Syncpoint trigger conditions set through VI_INCR_SYNCPT
 353                  * register are added to HW syncpt FIFO and when HW triggers,
 354                  * syncpt condition is removed from the FIFO and counter at
 355                  * syncpoint index will be incremented by the hardware and
 356                  * software can wait for counter to reach threshold to
 357                  * synchronize capturing frame with hardware capture events.
 358                  */
 359
 360                 /* increase channel syncpoint threshold for FRAME_START */
 361                 thresh = host1x_syncpt_incr_max(chan->frame_start_sp[i], 1);
 362                 fs_thresh[i] = thresh;
 363
 364                 /* Program FRAME_START trigger condition syncpt request */
 365                 frame_start = VI_CSI_PP_FRAME_START(portnos[i]);
 366                 value = VI_CFG_VI_INCR_SYNCPT_COND(frame_start) |
 367                         host1x_syncpt_id(chan->frame_start_sp[i]);
 368                 tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 369
 370                 /* increase channel syncpoint threshold for MW_ACK_DONE */
 371                 thresh = host1x_syncpt_incr_max(chan->mw_ack_sp[i], 1);
 372                 buf->mw_ack_sp_thresh[i] = thresh;
 373
 374                 /* Program MW_ACK_DONE trigger condition syncpt request */
 375                 mw_ack_done = VI_CSI_MW_ACK_DONE(portnos[i]);
 376                 value = VI_CFG_VI_INCR_SYNCPT_COND(mw_ack_done) |
 377                         host1x_syncpt_id(chan->mw_ack_sp[i]);
 378                 tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 379         }
 380
 381         /* enable single shot capture after all ganged ports are ready */
 382         for (i = 0; i < chan->numgangports; i++)
 383                 vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_SINGLE_SHOT,
 384                              SINGLE_SHOT_CAPTURE);
 385
 386         for (i = 0; i < chan->numgangports; i++) {
 387                 /*
 388                  * Wait for syncpt counter to reach frame start event threshold
 389                  */
 390                 err = host1x_syncpt_wait(chan->frame_start_sp[i], fs_thresh[i],
 391                                          TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 392                 if (err) {
 393                         capture_timedout = true;
 394                         /* increment syncpoint counter for timedout events */
 395                         host1x_syncpt_incr(chan->frame_start_sp[i]);
 396                         spin_lock(&chan->sp_incr_lock[i]);
 397                         host1x_syncpt_incr(chan->mw_ack_sp[i]);
 398                         spin_unlock(&chan->sp_incr_lock[i]);
 399                         /* clear errors and recover */
 400                         tegra_channel_capture_error_recover(chan, portnos[i]);
 401                 }
 402         }
 403
 404         if (capture_timedout) {
 405                 dev_err_ratelimited(&chan->video.dev,
 406                                     "frame start syncpt timeout: %d\n", err);
 407                 release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
 408                 return err;
 409         }
 410
 411         /* move buffer to capture done queue */
 412         spin_lock(&chan->done_lock);
 413         list_add_tail(&buf->queue, &chan->done);
 414         spin_unlock(&chan->done_lock);
 415
 416         /* wait up kthread for capture done */
 417         wake_up_interruptible(&chan->done_wait);
 418
 419         return 0;
 420 }
 421
 422 static void tegra_channel_capture_done(struct tegra_vi_channel *chan,
 423                                        struct tegra_channel_buffer *buf)
 424 {
 425         enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
 426         u32 value;
 427         bool capture_timedout = false;
 428         int ret, i;
 429
 430         for (i = 0; i < chan->numgangports; i++) {
 431                 /*
 432                  * Wait for syncpt counter to reach MW_ACK_DONE event threshold
 433                  */
 434                 ret = host1x_syncpt_wait(chan->mw_ack_sp[i],
 435                                          buf->mw_ack_sp_thresh[i],
 436                                          TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 437                 if (ret) {
 438                         capture_timedout = true;
 439                         state = VB2_BUF_STATE_ERROR;
 440                         /* increment syncpoint counter for timedout event */
 441                         spin_lock(&chan->sp_incr_lock[i]);
 442                         host1x_syncpt_incr(chan->mw_ack_sp[i]);
 443                         spin_unlock(&chan->sp_incr_lock[i]);
 444                 }
 445         }
 446
 447         if (capture_timedout)
 448                 dev_err_ratelimited(&chan->video.dev,
 449                                     "MW_ACK_DONE syncpt timeout: %d\n", ret);
 450         release_buffer(chan, buf, state);
 451 }
 452
 453 static int chan_capture_kthread_start(void *data)
 454 {
 455         struct tegra_vi_channel *chan = data;
 456         struct tegra_channel_buffer *buf;
 457         unsigned int retries = 0;
 458         int err = 0;
 459
 460         while (1) {
 461                 /*
 462                  * Source is not streaming if error is non-zero.
 463                  * So, do not dequeue buffers on error and let the thread sleep
 464                  * till kthread stop signal is received.
 465                  */
 466                 wait_event_interruptible(chan->start_wait,
 467                                          kthread_should_stop() ||
 468                                          (!list_empty(&chan->capture) &&
 469                                          !err));
 470
 471                 if (kthread_should_stop())
 472                         break;
 473
 474                 /* dequeue the buffer and start capture */
 475                 spin_lock(&chan->start_lock);
 476                 if (list_empty(&chan->capture)) {
 477                         spin_unlock(&chan->start_lock);
 478                         continue;
 479                 }
 480
 481                 buf = list_first_entry(&chan->capture,
 482                                        struct tegra_channel_buffer, queue);
 483                 list_del_init(&buf->queue);
 484                 spin_unlock(&chan->start_lock);
 485
 486                 err = tegra_channel_capture_frame(chan, buf);
 487                 if (!err) {
 488                         retries = 0;
 489                         continue;
 490                 }
 491
 492                 if (retries++ > chan->syncpt_timeout_retry)
 493                         vb2_queue_error(&chan->queue);
 494                 else
 495                         err = 0;
 496         }
 497
 498         return 0;
 499 }
 500
 501 static int chan_capture_kthread_finish(void *data)
 502 {
 503         struct tegra_vi_channel *chan = data;
 504         struct tegra_channel_buffer *buf;
 505
 506         while (1) {
 507                 wait_event_interruptible(chan->done_wait,
 508                                          !list_empty(&chan->done) ||
 509                                          kthread_should_stop());
 510
 511                 /* dequeue buffers and finish capture */
 512                 buf = dequeue_buf_done(chan);
 513                 while (buf) {
 514                         tegra_channel_capture_done(chan, buf);
 515                         buf = dequeue_buf_done(chan);
 516                 }
 517
 518                 if (kthread_should_stop())
 519                         break;
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int tegra210_vi_start_streaming(struct vb2_queue *vq, u32 count)
 526 {
 527         struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 528         struct media_pipeline *pipe = &chan->video.pipe;
 529         u32 val;
 530         u8 *portnos = chan->portnos;
 531         int ret, i;
 532
 533         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 534
 535         /* clear syncpt errors */
 536         val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 537         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 538
 539         /*
 540          * Sync point FIFO full stalls the host interface.
 541          * Setting NO_STALL will drop INCR_SYNCPT methods when fifos are
 542          * full and the corresponding condition bits in INCR_SYNCPT_ERROR
 543          * register will be set.
 544          * This allows SW to process error recovery.
 545          */
 546         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL,
 547                        VI_INCR_SYNCPT_NO_STALL);
 548
 549         /* start the pipeline */
 550         ret = media_pipeline_start(&chan->video.entity, pipe);
 551         if (ret < 0)
 552                 goto error_pipeline_start;
 553
 554         /* clear csi errors and do capture setup for all ports in gang mode */
 555         for (i = 0; i < chan->numgangports; i++) {
 556                 val = vi_csi_read(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS);
 557                 vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS, val);
 558
 559                 tegra_channel_capture_setup(chan, portnos[i]);
 560         }
 561
 562         ret = tegra_channel_set_stream(chan, true);
 563         if (ret < 0)
 564                 goto error_set_stream;
 565
 566         chan->sequence = 0;
 567
 568         /* start kthreads to capture data to buffer and return them */
 569         chan->kthread_start_capture = kthread_run(chan_capture_kthread_start,
 570                                                   chan, "%s:0",
 571                                                   chan->video.name);
 572         if (IS_ERR(chan->kthread_start_capture)) {
 573                 ret = PTR_ERR(chan->kthread_start_capture);
 574                 chan->kthread_start_capture = NULL;
 575                 dev_err(&chan->video.dev,
 576                         "failed to run capture start kthread: %d\n", ret);
 577                 goto error_kthread_start;
 578         }
 579
 580         chan->kthread_finish_capture = kthread_run(chan_capture_kthread_finish,
 581                                                    chan, "%s:1",
 582                                                    chan->video.name);
 583         if (IS_ERR(chan->kthread_finish_capture)) {
 584                 ret = PTR_ERR(chan->kthread_finish_capture);
 585                 chan->kthread_finish_capture = NULL;
 586                 dev_err(&chan->video.dev,
 587                         "failed to run capture finish kthread: %d\n", ret);
 588                 goto error_kthread_done;
 589         }
 590
 591         return 0;
 592
 593 error_kthread_done:
 594         kthread_stop(chan->kthread_start_capture);
 595 error_kthread_start:
 596         tegra_channel_set_stream(chan, false);
 597 error_set_stream:
 598         media_pipeline_stop(&chan->video.entity);
 599 error_pipeline_start:
 600         tegra_channel_release_buffers(chan, VB2_BUF_STATE_QUEUED);
 601         return ret;
 602 }
 603
 604 static void tegra210_vi_stop_streaming(struct vb2_queue *vq)
 605 {
 606         struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 607
 608         if (chan->kthread_start_capture) {
 609                 kthread_stop(chan->kthread_start_capture);
 610                 chan->kthread_start_capture = NULL;
 611         }
 612
 613         if (chan->kthread_finish_capture) {
 614                 kthread_stop(chan->kthread_finish_capture);
 615                 chan->kthread_finish_capture = NULL;
 616         }
 617
 618         tegra_channel_release_buffers(chan, VB2_BUF_STATE_ERROR);
 619         tegra_channel_set_stream(chan, false);
 620         media_pipeline_stop(&chan->video.entity);
 621 }
 622
 623 /*
 624  * Tegra210 VI Pixel memory format enum.
 625  * These format enum value gets programmed into corresponding Tegra VI
 626  * channel register bits.
 627  */
 628 enum tegra210_image_format {
 629         TEGRA210_IMAGE_FORMAT_T_L8 = 16,
 630
 631         TEGRA210_IMAGE_FORMAT_T_R16_I = 32,
 632         TEGRA210_IMAGE_FORMAT_T_B5G6R5,
 633         TEGRA210_IMAGE_FORMAT_T_R5G6B5,
 634         TEGRA210_IMAGE_FORMAT_T_A1B5G5R5,
 635         TEGRA210_IMAGE_FORMAT_T_A1R5G5B5,
 636         TEGRA210_IMAGE_FORMAT_T_B5G5R5A1,
 637         TEGRA210_IMAGE_FORMAT_T_R5G5B5A1,
 638         TEGRA210_IMAGE_FORMAT_T_A4B4G4R4,
 639         TEGRA210_IMAGE_FORMAT_T_A4R4G4B4,
 640         TEGRA210_IMAGE_FORMAT_T_B4G4R4A4,
 641         TEGRA210_IMAGE_FORMAT_T_R4G4B4A4,
 642
 643         TEGRA210_IMAGE_FORMAT_T_A8B8G8R8 = 64,
 644         TEGRA210_IMAGE_FORMAT_T_A8R8G8B8,
 645         TEGRA210_IMAGE_FORMAT_T_B8G8R8A8,
 646         TEGRA210_IMAGE_FORMAT_T_R8G8B8A8,
 647         TEGRA210_IMAGE_FORMAT_T_A2B10G10R10,
 648         TEGRA210_IMAGE_FORMAT_T_A2R10G10B10,
 649         TEGRA210_IMAGE_FORMAT_T_B10G10R10A2,
 650         TEGRA210_IMAGE_FORMAT_T_R10G10B10A2,
 651
 652         TEGRA210_IMAGE_FORMAT_T_A8Y8U8V8 = 193,
 653         TEGRA210_IMAGE_FORMAT_T_V8U8Y8A8,
 654
 655         TEGRA210_IMAGE_FORMAT_T_A2Y10U10V10 = 197,
 656         TEGRA210_IMAGE_FORMAT_T_V10U10Y10A2,
 657         TEGRA210_IMAGE_FORMAT_T_Y8_U8__Y8_V8,
 658         TEGRA210_IMAGE_FORMAT_T_Y8_V8__Y8_U8,
 659         TEGRA210_IMAGE_FORMAT_T_U8_Y8__V8_Y8,
 660         TEGRA210_IMAGE_FORMAT_T_V8_Y8__U8_Y8,
 661
 662         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N444 = 224,
 663         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N444,
 664         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N444,
 665         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N422,
 666         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N422,
 667         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N422,
 668         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N420,
 669         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N420,
 670         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N420,
 671         TEGRA210_IMAGE_FORMAT_T_X2LC10LB10LA10,
 672         TEGRA210_IMAGE_FORMAT_T_A2R6R6R6R6R6,
 673 };
 674
 675 #define TEGRA210_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP,        \
 676                            FORMAT, FOURCC)                              \
 677 {                                                                       \
 678         TEGRA_IMAGE_DT_##DATA_TYPE,                                     \
 679         BIT_WIDTH,                                                      \
 680         MEDIA_BUS_FMT_##MBUS_CODE,                                      \
 681         BPP,                                                            \
 682         TEGRA210_IMAGE_FORMAT_##FORMAT,                                 \
 683         V4L2_PIX_FMT_##FOURCC,                                          \
 684 }
 685
 686 /* Tegra210 supported video formats */
 687 static const struct tegra_video_format tegra210_video_formats[] = {
 688         /* RAW 8 */
 689         TEGRA210_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 1, T_L8, SRGGB8),
 690         TEGRA210_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 1, T_L8, SGRBG8),
 691         TEGRA210_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 1, T_L8, SGBRG8),
 692         TEGRA210_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 1, T_L8, SBGGR8),
 693         /* RAW 10 */
 694         TEGRA210_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, T_R16_I, SRGGB10),
 695         TEGRA210_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, T_R16_I, SGRBG10),
 696         TEGRA210_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, T_R16_I, SGBRG10),
 697         TEGRA210_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, T_R16_I, SBGGR10),
 698         /* RAW 12 */
 699         TEGRA210_VIDEO_FMT(RAW12, 12, SRGGB12_1X12, 2, T_R16_I, SRGGB12),
 700         TEGRA210_VIDEO_FMT(RAW12, 12, SGRBG12_1X12, 2, T_R16_I, SGRBG12),
 701         TEGRA210_VIDEO_FMT(RAW12, 12, SGBRG12_1X12, 2, T_R16_I, SGBRG12),
 702         TEGRA210_VIDEO_FMT(RAW12, 12, SBGGR12_1X12, 2, T_R16_I, SBGGR12),
 703         /* RGB888 */
 704         TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X24, 4, T_A8R8G8B8, XBGR32),
 705         TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X32_PADHI, 4, T_A8B8G8R8,
 706                            RGBX32),
 707         /* YUV422 */
 708         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, T_U8_Y8__V8_Y8, YVYU),
 709         TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, T_V8_Y8__U8_Y8, YUYV),
 710         TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, T_Y8_U8__Y8_V8, VYUY),
 711         TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, T_Y8_V8__Y8_U8, UYVY),
 712         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, T_Y8__V8U8_N422, NV16),
 713         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, T_U8_Y8__V8_Y8, YVYU),
 714         TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, T_V8_Y8__U8_Y8, YUYV),
 715         TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, T_Y8_U8__Y8_V8, VYUY),
 716         TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, T_Y8_V8__Y8_U8, UYVY),
 717 };
 718
 719 /* Tegra210 VI operations */
 720 static const struct tegra_vi_ops tegra210_vi_ops = {
 721         .vi_start_streaming = tegra210_vi_start_streaming,
 722         .vi_stop_streaming = tegra210_vi_stop_streaming,
 723 };
 724
 725 /* Tegra210 VI SoC data */
 726 const struct tegra_vi_soc tegra210_vi_soc = {
 727         .video_formats = tegra210_video_formats,
 728         .nformats = ARRAY_SIZE(tegra210_video_formats),
 729         .ops = &tegra210_vi_ops,
 730         .hw_revision = 3,
 731         .vi_max_channels = 6,
 732 #if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
 733         .vi_max_clk_hz = 499200000,
 734 #else
 735         .vi_max_clk_hz = 998400000,
 736 #endif
 737 };
 738
 739 /* Tegra210 CSI PHY registers accessors */
 740 static void csi_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 741                       u32 val)
 742 {
 743         void __iomem *csi_pp_base;
 744
 745         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 746
 747         writel_relaxed(val, csi_pp_base + addr);
 748 }
 749
 750 /* Tegra210 CSI Pixel parser registers accessors */
 751 static void pp_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 752 {
 753         void __iomem *csi_pp_base;
 754         unsigned int offset;
 755
 756         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 757         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 758
 759         writel_relaxed(val, csi_pp_base + offset + addr);
 760 }
 761
 762 static u32 pp_read(struct tegra_csi *csi, u8 portno, u32 addr)
 763 {
 764         void __iomem *csi_pp_base;
 765         unsigned int offset;
 766
 767         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 768         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 769
 770         return readl_relaxed(csi_pp_base + offset + addr);
 771 }
 772
 773 /* Tegra210 CSI CIL A/B port registers accessors */
 774 static void cil_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 775 {
 776         void __iomem *csi_cil_base;
 777         unsigned int offset;
 778
 779         csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 780                        TEGRA210_CSI_CIL_OFFSET;
 781         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 782
 783         writel_relaxed(val, csi_cil_base + offset + addr);
 784 }
 785
 786 static u32 cil_read(struct tegra_csi *csi, u8 portno, u32 addr)
 787 {
 788         void __iomem *csi_cil_base;
 789         unsigned int offset;
 790
 791         csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 792                        TEGRA210_CSI_CIL_OFFSET;
 793         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 794
 795         return readl_relaxed(csi_cil_base + offset + addr);
 796 }
 797
 798 /* Tegra210 CSI Test pattern generator registers accessor */
 799 static void tpg_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 800                       u32 val)
 801 {
 802         void __iomem *csi_pp_base;
 803         unsigned int offset;
 804
 805         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 806         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET +
 807                  TEGRA210_CSI_TPG_OFFSET;
 808
 809         writel_relaxed(val, csi_pp_base + offset + addr);
 810 }
 811
 812 /*
 813  * Tegra210 CSI operations
 814  */
 815 static void tegra210_csi_port_recover(struct tegra_csi_channel *csi_chan,
 816                                       u8 portno)
 817 {
 818         struct tegra_csi *csi = csi_chan->csi;
 819         u32 val;
 820
 821         /*
 822          * Recover CSI hardware in case of capture errors by issuing
 823          * software reset to CSICIL sensor, pixel parser, and clear errors
 824          * to have clean capture on  next streaming.
 825          */
 826         val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
 827         dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
 828
 829         val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
 830         dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
 831
 832         val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
 833         dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
 834
 835         if (csi_chan->numlanes == 4) {
 836                 /* reset CSI CIL sensor */
 837                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 838                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 839                 /*
 840                  * SW_STATUS_RESET resets all status bits of PPA, PPB, CILA,
 841                  * CILB status registers and debug counters.
 842                  * So, SW_STATUS_RESET can be used only when CSI brick is in
 843                  * x4 mode.
 844                  */
 845                 csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x1);
 846
 847                 /* sleep for 20 clock cycles to drain the FIFO */
 848                 usleep_range(10, 20);
 849
 850                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 851                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 852                 csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x0);
 853         } else {
 854                 /* reset CSICIL sensor */
 855                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 856                 usleep_range(10, 20);
 857                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 858
 859                 /* clear the errors */
 860                 pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS,
 861                          0xffffffff);
 862                 cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 863                 cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 864         }
 865 }
 866
 867 static void tegra210_csi_error_recover(struct tegra_csi_channel *csi_chan)
 868 {
 869         u8 *portnos = csi_chan->csi_port_nums;
 870         int i;
 871
 872         for (i = 0; i < csi_chan->numgangports; i++)
 873                 tegra210_csi_port_recover(csi_chan, portnos[i]);
 874 }
 875
 876 static int
 877 tegra210_csi_port_start_streaming(struct tegra_csi_channel *csi_chan,
 878                                   u8 portno)
 879 {
 880         struct tegra_csi *csi = csi_chan->csi;
 881         u8 clk_settle_time = 0;
 882         u8 ths_settle_time = 10;
 883         u32 val;
 884
 885         if (!csi_chan->pg_mode)
 886                 tegra_csi_calc_settle_time(csi_chan, portno, &clk_settle_time,
 887                                            &ths_settle_time);
 888
 889         csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
 890
 891         /* clean up status */
 892         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, 0xffffffff);
 893         cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 894         cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 895         cil_write(csi, portno, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
 896
 897         /* CIL PHY registers setup */
 898         cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
 899         cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL,
 900                   FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
 901                   FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
 902
 903         /*
 904          * The CSI unit provides for connection of up to six cameras in
 905          * the system and is organized as three identical instances of
 906          * two MIPI support blocks, each with a separate 4-lane
 907          * interface that can be configured as a single camera with 4
 908          * lanes or as a dual camera with 2 lanes available for each
 909          * camera.
 910          */
 911         if (csi_chan->numlanes == 4) {
 912                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 913                 cil_write(csi, portno + 1, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 914                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
 915
 916                 cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0,
 917                           BRICK_CLOCK_A_4X);
 918                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
 919                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
 920                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL,
 921                           FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
 922                           FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
 923                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
 924                           CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
 925         } else {
 926                 val = ((portno & 1) == PORT_A) ?
 927                       CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_NOP :
 928                       CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_NOP;
 929                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
 930         }
 931
 932         /* CSI pixel parser registers setup */
 933         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
 934                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
 935                  CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_RST);
 936         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0x0);
 937         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL0,
 938                  CSI_PP_PACKET_HEADER_SENT |
 939                  CSI_PP_DATA_IDENTIFIER_ENABLE |
 940                  CSI_PP_WORD_COUNT_SELECT_HEADER |
 941                  CSI_PP_CRC_CHECK_ENABLE |  CSI_PP_WC_CHECK |
 942                  CSI_PP_OUTPUT_FORMAT_STORE | CSI_PPA_PAD_LINE_NOPAD |
 943                  CSI_PP_HEADER_EC_DISABLE | CSI_PPA_PAD_FRAME_NOPAD |
 944                  (portno & 1));
 945         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL1,
 946                  (0x1 << CSI_PP_TOP_FIELD_FRAME_OFFSET) |
 947                  (0x1 << CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET));
 948         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_GAP,
 949                  0x14 << PP_FRAME_MIN_GAP_OFFSET);
 950         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME, 0x0);
 951         pp_write(csi, portno, TEGRA_CSI_INPUT_STREAM_CONTROL,
 952                  (0x3f << CSI_SKIP_PACKET_THRESHOLD_OFFSET) |
 953                  (csi_chan->numlanes - 1));
 954
 955         /* TPG setup */
 956         if (csi_chan->pg_mode) {
 957                 tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
 958                           ((csi_chan->pg_mode - 1) << PG_MODE_OFFSET) |
 959                           PG_ENABLE);
 960                 tpg_write(csi, portno, TEGRA_CSI_PG_BLANK,
 961                           csi_chan->v_blank << PG_VBLANK_OFFSET |
 962                           csi_chan->h_blank);
 963                 tpg_write(csi, portno, TEGRA_CSI_PG_PHASE, 0x0);
 964                 tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ,
 965                           (0x10 << PG_RED_VERT_INIT_FREQ_OFFSET) |
 966                           (0x10 << PG_RED_HOR_INIT_FREQ_OFFSET));
 967                 tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ_RATE, 0x0);
 968                 tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ,
 969                           (0x10 << PG_GREEN_VERT_INIT_FREQ_OFFSET) |
 970                           (0x10 << PG_GREEN_HOR_INIT_FREQ_OFFSET));
 971                 tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ_RATE, 0x0);
 972                 tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ,
 973                           (0x10 << PG_BLUE_VERT_INIT_FREQ_OFFSET) |
 974                           (0x10 << PG_BLUE_HOR_INIT_FREQ_OFFSET));
 975                 tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ_RATE, 0x0);
 976         }
 977
 978         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
 979                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
 980                  CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_ENABLE);
 981
 982         return 0;
 983 }
 984
 985 static void
 986 tegra210_csi_port_stop_streaming(struct tegra_csi_channel *csi_chan, u8 portno)
 987 {
 988         struct tegra_csi *csi = csi_chan->csi;
 989         u32 val;
 990
 991         val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
 992
 993         dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
 994         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, val);
 995
 996         val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
 997         dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
 998         cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, val);
 999
1000         val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
1001         dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
1002         cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, val);
1003
1004         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1005                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1006                  CSI_PP_DISABLE);
1007
1008         if (csi_chan->pg_mode) {
1009                 tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
1010                           PG_DISABLE);
1011                 return;
1012         }
1013
1014         if (csi_chan->numlanes == 4) {
1015                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
1016                           CSI_A_PHY_CIL_DISABLE |
1017                           CSI_B_PHY_CIL_DISABLE);
1018         } else {
1019                 val = ((portno & 1) == PORT_A) ?
1020                       CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
1021                       CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
1022                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
1023         }
1024 }
1025
1026 static int tegra210_csi_start_streaming(struct tegra_csi_channel *csi_chan)
1027 {
1028         u8 *portnos = csi_chan->csi_port_nums;
1029         int ret, i;
1030
1031         for (i = 0; i < csi_chan->numgangports; i++) {
1032                 ret = tegra210_csi_port_start_streaming(csi_chan, portnos[i]);
1033                 if (ret)
1034                         goto stream_start_fail;
1035         }
1036
1037         return 0;
1038
1039 stream_start_fail:
1040         for (i = i - 1; i >= 0; i--)
1041                 tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1042
1043         return ret;
1044 }
1045
1046 static void tegra210_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
1047 {
1048         u8 *portnos = csi_chan->csi_port_nums;
1049         int i;
1050
1051         for (i = 0; i < csi_chan->numgangports; i++)
1052                 tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1053 }
1054
1055 /*
1056  * Tegra210 CSI TPG frame rate table with horizontal and vertical
1057  * blanking intervals for corresponding format and resolution.
1058  * Blanking intervals are tuned values from design team for max TPG
1059  * clock rate.
1060  */
1061 static const struct tpg_framerate tegra210_tpg_frmrate_table[] = {
1062         {
1063                 .frmsize = { 1280, 720 },
1064                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1065                 .framerate = 120,
1066                 .h_blank = 512,
1067                 .v_blank = 8,
1068         },
1069         {
1070                 .frmsize = { 1920, 1080 },
1071                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1072                 .framerate = 60,
1073                 .h_blank = 512,
1074                 .v_blank = 8,
1075         },
1076         {
1077                 .frmsize = { 3840, 2160 },
1078                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1079                 .framerate = 20,
1080                 .h_blank = 8,
1081                 .v_blank = 8,
1082         },
1083         {
1084                 .frmsize = { 1280, 720 },
1085                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1086                 .framerate = 60,
1087                 .h_blank = 512,
1088                 .v_blank = 8,
1089         },
1090         {
1091                 .frmsize = { 1920, 1080 },
1092                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1093                 .framerate = 30,
1094                 .h_blank = 512,
1095                 .v_blank = 8,
1096         },
1097         {
1098                 .frmsize = { 3840, 2160 },
1099                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1100                 .framerate = 8,
1101                 .h_blank = 8,
1102                 .v_blank = 8,
1103         },
1104 };
1105
1106 static const char * const tegra210_csi_cil_clks[] = {
1107         "csi",
1108         "cilab",
1109         "cilcd",
1110         "cile",
1111 #if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
1112         "csi_tpg",
1113 #endif
1114 };
1115
1116 /* Tegra210 CSI operations */
1117 static const struct tegra_csi_ops tegra210_csi_ops = {
1118         .csi_start_streaming = tegra210_csi_start_streaming,
1119         .csi_stop_streaming = tegra210_csi_stop_streaming,
1120         .csi_err_recover = tegra210_csi_error_recover,
1121 };
1122
1123 /* Tegra210 CSI SoC data */
1124 const struct tegra_csi_soc tegra210_csi_soc = {
1125         .ops = &tegra210_csi_ops,
1126         .csi_max_channels = 6,
1127         .clk_names = tegra210_csi_cil_clks,
1128         .num_clks = ARRAY_SIZE(tegra210_csi_cil_clks),
1129         .tpg_frmrate_table = tegra210_tpg_frmrate_table,
1130         .tpg_frmrate_table_size = ARRAY_SIZE(tegra210_tpg_frmrate_table),
1131 };