drivers/staging/media/tegra-video/tegra210.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 /*
   7  * This source file contains Tegra210 supported video formats,
   8  * VI and CSI SoC specific data, operations and registers accessors.
   9  */
  10 #include <linux/bitfield.h>
  11 #include <linux/clk.h>
  12 #include <linux/clk/tegra.h>
  13 #include <linux/delay.h>
  14 #include <linux/host1x.h>
  15 #include <linux/kthread.h>
  16
  17 #include "csi.h"
  18 #include "vi.h"
  19
  20 #define TEGRA210_MIN_WIDTH      32U
  21 #define TEGRA210_MAX_WIDTH      32768U
  22 #define TEGRA210_MIN_HEIGHT     32U
  23 #define TEGRA210_MAX_HEIGHT     32768U
  24
  25 #define SURFACE_ALIGN_BYTES     64
  26
  27 #define TEGRA_VI_SYNCPT_WAIT_TIMEOUT                    msecs_to_jiffies(200)
  28
  29 /* Tegra210 VI registers */
  30 #define TEGRA_VI_CFG_VI_INCR_SYNCPT                     0x000
  31 #define   VI_CFG_VI_INCR_SYNCPT_COND(x)                 (((x) & 0xff) << 8)
  32 #define   VI_CSI_PP_FRAME_START(port)                   (5 + (port) * 4)
  33 #define   VI_CSI_MW_ACK_DONE(port)                      (7 + (port) * 4)
  34 #define TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL               0x004
  35 #define   VI_INCR_SYNCPT_NO_STALL                       BIT(8)
  36 #define TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR               0x008
  37 #define TEGRA_VI_CFG_CG_CTRL                            0x0b8
  38 #define   VI_CG_2ND_LEVEL_EN                            0x1
  39
  40 /* Tegra210 VI CSI registers */
  41 #define TEGRA_VI_CSI_SW_RESET                           0x000
  42 #define TEGRA_VI_CSI_SINGLE_SHOT                        0x004
  43 #define   SINGLE_SHOT_CAPTURE                           0x1
  44 #define TEGRA_VI_CSI_IMAGE_DEF                          0x00c
  45 #define   BYPASS_PXL_TRANSFORM_OFFSET                   24
  46 #define   IMAGE_DEF_FORMAT_OFFSET                       16
  47 #define   IMAGE_DEF_DEST_MEM                            0x1
  48 #define TEGRA_VI_CSI_IMAGE_SIZE                         0x018
  49 #define   IMAGE_SIZE_HEIGHT_OFFSET                      16
  50 #define TEGRA_VI_CSI_IMAGE_SIZE_WC                      0x01c
  51 #define TEGRA_VI_CSI_IMAGE_DT                           0x020
  52 #define TEGRA_VI_CSI_SURFACE0_OFFSET_MSB                0x024
  53 #define TEGRA_VI_CSI_SURFACE0_OFFSET_LSB                0x028
  54 #define TEGRA_VI_CSI_SURFACE1_OFFSET_MSB                0x02c
  55 #define TEGRA_VI_CSI_SURFACE1_OFFSET_LSB                0x030
  56 #define TEGRA_VI_CSI_SURFACE2_OFFSET_MSB                0x034
  57 #define TEGRA_VI_CSI_SURFACE2_OFFSET_LSB                0x038
  58 #define TEGRA_VI_CSI_SURFACE0_STRIDE                    0x054
  59 #define TEGRA_VI_CSI_SURFACE1_STRIDE                    0x058
  60 #define TEGRA_VI_CSI_SURFACE2_STRIDE                    0x05c
  61 #define TEGRA_VI_CSI_SURFACE_HEIGHT0                    0x060
  62 #define TEGRA_VI_CSI_ERROR_STATUS                       0x084
  63
  64 /* Tegra210 CSI Pixel Parser registers: Starts from 0x838, offset 0x0 */
  65 #define TEGRA_CSI_INPUT_STREAM_CONTROL                  0x000
  66 #define   CSI_SKIP_PACKET_THRESHOLD_OFFSET              16
  67 #define TEGRA_CSI_PIXEL_STREAM_CONTROL0                 0x004
  68 #define   CSI_PP_PACKET_HEADER_SENT                     BIT(4)
  69 #define   CSI_PP_DATA_IDENTIFIER_ENABLE                 BIT(5)
  70 #define   CSI_PP_WORD_COUNT_SELECT_HEADER               BIT(6)
  71 #define   CSI_PP_CRC_CHECK_ENABLE                       BIT(7)
  72 #define   CSI_PP_WC_CHECK                               BIT(8)
  73 #define   CSI_PP_OUTPUT_FORMAT_STORE                    (0x3 << 16)
  74 #define   CSI_PPA_PAD_LINE_NOPAD                        (0x2 << 24)
  75 #define   CSI_PP_HEADER_EC_DISABLE                      (0x1 << 27)
  76 #define   CSI_PPA_PAD_FRAME_NOPAD                       (0x2 << 28)
  77 #define TEGRA_CSI_PIXEL_STREAM_CONTROL1                 0x008
  78 #define   CSI_PP_TOP_FIELD_FRAME_OFFSET                 0
  79 #define   CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET            4
  80 #define TEGRA_CSI_PIXEL_STREAM_GAP                      0x00c
  81 #define   PP_FRAME_MIN_GAP_OFFSET                       16
  82 #define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND               0x010
  83 #define   CSI_PP_ENABLE                                 0x1
  84 #define   CSI_PP_DISABLE                                0x2
  85 #define   CSI_PP_RST                                    0x3
  86 #define   CSI_PP_SINGLE_SHOT_ENABLE                     (0x1 << 2)
  87 #define   CSI_PP_START_MARKER_FRAME_MAX_OFFSET          12
  88 #define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME           0x014
  89 #define TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK           0x018
  90 #define TEGRA_CSI_PIXEL_PARSER_STATUS                   0x01c
  91
  92 /* Tegra210 CSI PHY registers */
  93 /* CSI_PHY_CIL_COMMAND_0 offset 0x0d0 from TEGRA_CSI_PIXEL_PARSER_0_BASE */
  94 #define TEGRA_CSI_PHY_CIL_COMMAND                       0x0d0
  95 #define   CSI_A_PHY_CIL_NOP                             0x0
  96 #define   CSI_A_PHY_CIL_ENABLE                          0x1
  97 #define   CSI_A_PHY_CIL_DISABLE                         0x2
  98 #define   CSI_A_PHY_CIL_ENABLE_MASK                     0x3
  99 #define   CSI_B_PHY_CIL_NOP                             (0x0 << 8)
 100 #define   CSI_B_PHY_CIL_ENABLE                          (0x1 << 8)
 101 #define   CSI_B_PHY_CIL_DISABLE                         (0x2 << 8)
 102 #define   CSI_B_PHY_CIL_ENABLE_MASK                     (0x3 << 8)
 103
 104 #define TEGRA_CSI_CIL_PAD_CONFIG0                       0x000
 105 #define   BRICK_CLOCK_A_4X                              (0x1 << 16)
 106 #define   BRICK_CLOCK_B_4X                              (0x2 << 16)
 107 #define TEGRA_CSI_CIL_PAD_CONFIG1                       0x004
 108 #define TEGRA_CSI_CIL_PHY_CONTROL                       0x008
 109 #define   CLK_SETTLE_MASK                               GENMASK(13, 8)
 110 #define   THS_SETTLE_MASK                               GENMASK(5, 0)
 111 #define TEGRA_CSI_CIL_INTERRUPT_MASK                    0x00c
 112 #define TEGRA_CSI_CIL_STATUS                            0x010
 113 #define TEGRA_CSI_CILX_STATUS                           0x014
 114 #define TEGRA_CSI_CIL_SW_SENSOR_RESET                   0x020
 115
 116 #define TEGRA_CSI_PATTERN_GENERATOR_CTRL                0x000
 117 #define   PG_MODE_OFFSET                                2
 118 #define   PG_ENABLE                                     0x1
 119 #define   PG_DISABLE                                    0x0
 120 #define TEGRA_CSI_PG_BLANK                              0x004
 121 #define   PG_VBLANK_OFFSET                              16
 122 #define TEGRA_CSI_PG_PHASE                              0x008
 123 #define TEGRA_CSI_PG_RED_FREQ                           0x00c
 124 #define   PG_RED_VERT_INIT_FREQ_OFFSET                  16
 125 #define   PG_RED_HOR_INIT_FREQ_OFFSET                   0
 126 #define TEGRA_CSI_PG_RED_FREQ_RATE                      0x010
 127 #define TEGRA_CSI_PG_GREEN_FREQ                         0x014
 128 #define   PG_GREEN_VERT_INIT_FREQ_OFFSET                16
 129 #define   PG_GREEN_HOR_INIT_FREQ_OFFSET                 0
 130 #define TEGRA_CSI_PG_GREEN_FREQ_RATE                    0x018
 131 #define TEGRA_CSI_PG_BLUE_FREQ                          0x01c
 132 #define   PG_BLUE_VERT_INIT_FREQ_OFFSET                 16
 133 #define   PG_BLUE_HOR_INIT_FREQ_OFFSET                  0
 134 #define TEGRA_CSI_PG_BLUE_FREQ_RATE                     0x020
 135 #define TEGRA_CSI_PG_AOHDR                              0x024
 136 #define TEGRA_CSI_CSI_SW_STATUS_RESET                   0x214
 137 #define TEGRA_CSI_CLKEN_OVERRIDE                        0x218
 138
 139 #define TEGRA210_CSI_PORT_OFFSET                        0x34
 140 #define TEGRA210_CSI_CIL_OFFSET                         0x0f4
 141 #define TEGRA210_CSI_TPG_OFFSET                         0x18c
 142
 143 #define CSI_PP_OFFSET(block)                            ((block) * 0x800)
 144 #define TEGRA210_VI_CSI_BASE(x)                         (0x100 + (x) * 0x100)
 145
 146 /* Tegra210 VI registers accessors */
 147 static void tegra_vi_write(struct tegra_vi_channel *chan, unsigned int addr,
 148                            u32 val)
 149 {
 150         writel_relaxed(val, chan->vi->iomem + addr);
 151 }
 152
 153 static u32 tegra_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
 154 {
 155         return readl_relaxed(chan->vi->iomem + addr);
 156 }
 157
 158 /* Tegra210 VI_CSI registers accessors */
 159 static void vi_csi_write(struct tegra_vi_channel *chan, u8 portno,
 160                          unsigned int addr, u32 val)
 161 {
 162         void __iomem *vi_csi_base;
 163
 164         vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 165
 166         writel_relaxed(val, vi_csi_base + addr);
 167 }
 168
 169 static u32 vi_csi_read(struct tegra_vi_channel *chan, u8 portno,
 170                        unsigned int addr)
 171 {
 172         void __iomem *vi_csi_base;
 173
 174         vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 175
 176         return readl_relaxed(vi_csi_base + addr);
 177 }
 178
 179 /*
 180  * Tegra210 VI channel capture operations
 181  */
 182
 183 static int tegra210_channel_host1x_syncpt_init(struct tegra_vi_channel *chan)
 184 {
 185         struct tegra_vi *vi = chan->vi;
 186         unsigned long flags = HOST1X_SYNCPT_CLIENT_MANAGED;
 187         struct host1x_syncpt *fs_sp;
 188         struct host1x_syncpt *mw_sp;
 189         int ret, i;
 190
 191         for (i = 0; i < chan->numgangports; i++) {
 192                 fs_sp = host1x_syncpt_request(&vi->client, flags);
 193                 if (!fs_sp) {
 194                         dev_err(vi->dev, "failed to request frame start syncpoint\n");
 195                         ret = -ENOMEM;
 196                         goto free_syncpts;
 197                 }
 198
 199                 mw_sp = host1x_syncpt_request(&vi->client, flags);
 200                 if (!mw_sp) {
 201                         dev_err(vi->dev, "failed to request memory ack syncpoint\n");
 202                         host1x_syncpt_put(fs_sp);
 203                         ret = -ENOMEM;
 204                         goto free_syncpts;
 205                 }
 206
 207                 chan->frame_start_sp[i] = fs_sp;
 208                 chan->mw_ack_sp[i] = mw_sp;
 209                 spin_lock_init(&chan->sp_incr_lock[i]);
 210         }
 211
 212         return 0;
 213
 214 free_syncpts:
 215         for (i = 0; i < chan->numgangports; i++) {
 216                 host1x_syncpt_put(chan->mw_ack_sp[i]);
 217                 host1x_syncpt_put(chan->frame_start_sp[i]);
 218         }
 219         return ret;
 220 }
 221
 222 static void tegra210_channel_host1x_syncpt_free(struct tegra_vi_channel *chan)
 223 {
 224         int i;
 225
 226         for (i = 0; i < chan->numgangports; i++) {
 227                 host1x_syncpt_put(chan->mw_ack_sp[i]);
 228                 host1x_syncpt_put(chan->frame_start_sp[i]);
 229         }
 230 }
 231
 232 static void tegra210_fmt_align(struct v4l2_pix_format *pix, unsigned int bpp)
 233 {
 234         unsigned int min_bpl;
 235         unsigned int max_bpl;
 236         unsigned int bpl;
 237
 238         /*
 239          * The transfer alignment requirements are expressed in bytes.
 240          * Clamp the requested width and height to the limits.
 241          */
 242         pix->width = clamp(pix->width, TEGRA210_MIN_WIDTH, TEGRA210_MAX_WIDTH);
 243         pix->height = clamp(pix->height, TEGRA210_MIN_HEIGHT, TEGRA210_MAX_HEIGHT);
 244
 245         /* Clamp the requested bytes per line value. If the maximum bytes per
 246          * line value is zero, the module doesn't support user configurable
 247          * line sizes. Override the requested value with the minimum in that
 248          * case.
 249          */
 250         min_bpl = pix->width * bpp;
 251         max_bpl = rounddown(TEGRA210_MAX_WIDTH, SURFACE_ALIGN_BYTES);
 252         bpl = roundup(pix->bytesperline, SURFACE_ALIGN_BYTES);
 253
 254         pix->bytesperline = clamp(bpl, min_bpl, max_bpl);
 255         pix->sizeimage = pix->bytesperline * pix->height;
 256         if (pix->pixelformat == V4L2_PIX_FMT_NV16)
 257                 pix->sizeimage *= 2;
 258 }
 259
 260 static int tegra_channel_capture_setup(struct tegra_vi_channel *chan,
 261                                        u8 portno)
 262 {
 263         u32 height = chan->format.height;
 264         u32 width = chan->format.width;
 265         u32 format = chan->fmtinfo->img_fmt;
 266         u32 data_type = chan->fmtinfo->img_dt;
 267         u32 word_count = (width * chan->fmtinfo->bit_width) / 8;
 268         u32 bypass_pixel_transform = BIT(BYPASS_PXL_TRANSFORM_OFFSET);
 269
 270         /*
 271          * VI Pixel transformation unit converts source pixels data format
 272          * into selected destination pixel format and aligns properly while
 273          * interfacing with memory packer.
 274          * This pixel transformation should be enabled for YUV and RGB
 275          * formats and should be bypassed for RAW formats as RAW formats
 276          * only support direct to memory.
 277          */
 278         if (chan->pg_mode || data_type == TEGRA_IMAGE_DT_YUV422_8 ||
 279             data_type == TEGRA_IMAGE_DT_RGB888)
 280                 bypass_pixel_transform = 0;
 281
 282         /*
 283          * For x8 source streaming, the source image is split onto two x4 ports
 284          * with left half to first x4 port and right half to second x4 port.
 285          * So, use split width and corresponding word count for each x4 port.
 286          */
 287         if (chan->numgangports > 1) {
 288                 width = width >> 1;
 289                 word_count = (width * chan->fmtinfo->bit_width) / 8;
 290         }
 291
 292         vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, 0xffffffff);
 293         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DEF,
 294                      bypass_pixel_transform |
 295                      (format << IMAGE_DEF_FORMAT_OFFSET) |
 296                      IMAGE_DEF_DEST_MEM);
 297         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DT, data_type);
 298         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE_WC, word_count);
 299         vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE,
 300                      (height << IMAGE_SIZE_HEIGHT_OFFSET) | width);
 301         return 0;
 302 }
 303
 304 static void tegra_channel_vi_soft_reset(struct tegra_vi_channel *chan,
 305                                         u8 portno)
 306 {
 307         /* disable clock gating to enable continuous clock */
 308         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, 0);
 309         /*
 310          * Soft reset memory client interface, pixel format logic, sensor
 311          * control logic, and a shadow copy logic to bring VI to clean state.
 312          */
 313         vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0xf);
 314         usleep_range(100, 200);
 315         vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0x0);
 316
 317         /* enable back VI clock gating */
 318         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 319 }
 320
 321 static void tegra_channel_capture_error_recover(struct tegra_vi_channel *chan,
 322                                                 u8 portno)
 323 {
 324         struct v4l2_subdev *subdev;
 325         u32 val;
 326
 327         /*
 328          * Recover VI and CSI hardware blocks in case of missing frame start
 329          * events due to source not streaming or noisy csi inputs from the
 330          * external source or many outstanding frame start or MW_ACK_DONE
 331          * events which can cause CSI and VI hardware hang.
 332          * This helps to have a clean capture for next frame.
 333          */
 334         val = vi_csi_read(chan, portno, TEGRA_VI_CSI_ERROR_STATUS);
 335         dev_dbg(&chan->video.dev, "TEGRA_VI_CSI_ERROR_STATUS 0x%08x\n", val);
 336         vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, val);
 337
 338         val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 339         dev_dbg(&chan->video.dev,
 340                 "TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR 0x%08x\n", val);
 341         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 342
 343         /* recover VI by issuing software reset and re-setup for capture */
 344         tegra_channel_vi_soft_reset(chan, portno);
 345         tegra_channel_capture_setup(chan, portno);
 346
 347         /* recover CSI block */
 348         subdev = tegra_channel_get_remote_csi_subdev(chan);
 349         tegra_csi_error_recover(subdev);
 350 }
 351
 352 static struct tegra_channel_buffer *
 353 dequeue_buf_done(struct tegra_vi_channel *chan)
 354 {
 355         struct tegra_channel_buffer *buf = NULL;
 356
 357         spin_lock(&chan->done_lock);
 358         if (list_empty(&chan->done)) {
 359                 spin_unlock(&chan->done_lock);
 360                 return NULL;
 361         }
 362
 363         buf = list_first_entry(&chan->done,
 364                                struct tegra_channel_buffer, queue);
 365         if (buf)
 366                 list_del_init(&buf->queue);
 367         spin_unlock(&chan->done_lock);
 368
 369         return buf;
 370 }
 371
 372 static void release_buffer(struct tegra_vi_channel *chan,
 373                            struct tegra_channel_buffer *buf,
 374                            enum vb2_buffer_state state)
 375 {
 376         struct vb2_v4l2_buffer *vb = &buf->buf;
 377
 378         vb->sequence = chan->sequence++;
 379         vb->field = V4L2_FIELD_NONE;
 380         vb->vb2_buf.timestamp = ktime_get_ns();
 381         vb2_buffer_done(&vb->vb2_buf, state);
 382 }
 383
 384 static void tegra_channel_vi_buffer_setup(struct tegra_vi_channel *chan,
 385                                           u8 portno, u32 buf_offset,
 386                                           struct tegra_channel_buffer *buf)
 387 {
 388         int bytesperline = chan->format.bytesperline;
 389         u32 sizeimage = chan->format.sizeimage;
 390
 391         /* program buffer address by using surface 0 */
 392         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_MSB,
 393                      ((u64)buf->addr + buf_offset) >> 32);
 394         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_LSB,
 395                      buf->addr + buf_offset);
 396         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_STRIDE, bytesperline);
 397
 398         if (chan->fmtinfo->fourcc != V4L2_PIX_FMT_NV16)
 399                 return;
 400         /*
 401          * Program surface 1 for UV plane with offset sizeimage from Y plane.
 402          */
 403         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_MSB,
 404                      (((u64)buf->addr + sizeimage / 2) + buf_offset) >> 32);
 405         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_LSB,
 406                      buf->addr + sizeimage / 2 + buf_offset);
 407         vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_STRIDE, bytesperline);
 408 }
 409
 410 static int tegra_channel_capture_frame(struct tegra_vi_channel *chan,
 411                                        struct tegra_channel_buffer *buf)
 412 {
 413         u32 thresh, value, frame_start, mw_ack_done;
 414         u32 fs_thresh[GANG_PORTS_MAX];
 415         u8 *portnos = chan->portnos;
 416         int gang_bpl = (chan->format.width >> 1) * chan->fmtinfo->bpp;
 417         u32 buf_offset;
 418         bool capture_timedout = false;
 419         int err, i;
 420
 421         for (i = 0; i < chan->numgangports; i++) {
 422                 /*
 423                  * Align buffers side-by-side for all consecutive x4 ports
 424                  * in gang ports using bytes per line based on source split
 425                  * width.
 426                  */
 427                 buf_offset = i * roundup(gang_bpl, SURFACE_ALIGN_BYTES);
 428                 tegra_channel_vi_buffer_setup(chan, portnos[i], buf_offset,
 429                                               buf);
 430
 431                 /*
 432                  * Tegra VI block interacts with host1x syncpt to synchronize
 433                  * programmed condition and hardware operation for capture.
 434                  * Frame start and Memory write acknowledge syncpts has their
 435                  * own FIFO of depth 2.
 436                  *
 437                  * Syncpoint trigger conditions set through VI_INCR_SYNCPT
 438                  * register are added to HW syncpt FIFO and when HW triggers,
 439                  * syncpt condition is removed from the FIFO and counter at
 440                  * syncpoint index will be incremented by the hardware and
 441                  * software can wait for counter to reach threshold to
 442                  * synchronize capturing frame with hardware capture events.
 443                  */
 444
 445                 /* increase channel syncpoint threshold for FRAME_START */
 446                 thresh = host1x_syncpt_incr_max(chan->frame_start_sp[i], 1);
 447                 fs_thresh[i] = thresh;
 448
 449                 /* Program FRAME_START trigger condition syncpt request */
 450                 frame_start = VI_CSI_PP_FRAME_START(portnos[i]);
 451                 value = VI_CFG_VI_INCR_SYNCPT_COND(frame_start) |
 452                         host1x_syncpt_id(chan->frame_start_sp[i]);
 453                 tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 454
 455                 /* increase channel syncpoint threshold for MW_ACK_DONE */
 456                 thresh = host1x_syncpt_incr_max(chan->mw_ack_sp[i], 1);
 457                 buf->mw_ack_sp_thresh[i] = thresh;
 458
 459                 /* Program MW_ACK_DONE trigger condition syncpt request */
 460                 mw_ack_done = VI_CSI_MW_ACK_DONE(portnos[i]);
 461                 value = VI_CFG_VI_INCR_SYNCPT_COND(mw_ack_done) |
 462                         host1x_syncpt_id(chan->mw_ack_sp[i]);
 463                 tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 464         }
 465
 466         /* enable single shot capture after all ganged ports are ready */
 467         for (i = 0; i < chan->numgangports; i++)
 468                 vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_SINGLE_SHOT,
 469                              SINGLE_SHOT_CAPTURE);
 470
 471         for (i = 0; i < chan->numgangports; i++) {
 472                 /*
 473                  * Wait for syncpt counter to reach frame start event threshold
 474                  */
 475                 err = host1x_syncpt_wait(chan->frame_start_sp[i], fs_thresh[i],
 476                                          TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 477                 if (err) {
 478                         capture_timedout = true;
 479                         /* increment syncpoint counter for timedout events */
 480                         host1x_syncpt_incr(chan->frame_start_sp[i]);
 481                         spin_lock(&chan->sp_incr_lock[i]);
 482                         host1x_syncpt_incr(chan->mw_ack_sp[i]);
 483                         spin_unlock(&chan->sp_incr_lock[i]);
 484                         /* clear errors and recover */
 485                         tegra_channel_capture_error_recover(chan, portnos[i]);
 486                 }
 487         }
 488
 489         if (capture_timedout) {
 490                 dev_err_ratelimited(&chan->video.dev,
 491                                     "frame start syncpt timeout: %d\n", err);
 492                 release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
 493                 return err;
 494         }
 495
 496         /* move buffer to capture done queue */
 497         spin_lock(&chan->done_lock);
 498         list_add_tail(&buf->queue, &chan->done);
 499         spin_unlock(&chan->done_lock);
 500
 501         /* wait up kthread for capture done */
 502         wake_up_interruptible(&chan->done_wait);
 503
 504         return 0;
 505 }
 506
 507 static void tegra_channel_capture_done(struct tegra_vi_channel *chan,
 508                                        struct tegra_channel_buffer *buf)
 509 {
 510         enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
 511         u32 value;
 512         bool capture_timedout = false;
 513         int ret, i;
 514
 515         for (i = 0; i < chan->numgangports; i++) {
 516                 /*
 517                  * Wait for syncpt counter to reach MW_ACK_DONE event threshold
 518                  */
 519                 ret = host1x_syncpt_wait(chan->mw_ack_sp[i],
 520                                          buf->mw_ack_sp_thresh[i],
 521                                          TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 522                 if (ret) {
 523                         capture_timedout = true;
 524                         state = VB2_BUF_STATE_ERROR;
 525                         /* increment syncpoint counter for timedout event */
 526                         spin_lock(&chan->sp_incr_lock[i]);
 527                         host1x_syncpt_incr(chan->mw_ack_sp[i]);
 528                         spin_unlock(&chan->sp_incr_lock[i]);
 529                 }
 530         }
 531
 532         if (capture_timedout)
 533                 dev_err_ratelimited(&chan->video.dev,
 534                                     "MW_ACK_DONE syncpt timeout: %d\n", ret);
 535         release_buffer(chan, buf, state);
 536 }
 537
 538 static int chan_capture_kthread_start(void *data)
 539 {
 540         struct tegra_vi_channel *chan = data;
 541         struct tegra_channel_buffer *buf;
 542         unsigned int retries = 0;
 543         int err = 0;
 544
 545         while (1) {
 546                 /*
 547                  * Source is not streaming if error is non-zero.
 548                  * So, do not dequeue buffers on error and let the thread sleep
 549                  * till kthread stop signal is received.
 550                  */
 551                 wait_event_interruptible(chan->start_wait,
 552                                          kthread_should_stop() ||
 553                                          (!list_empty(&chan->capture) &&
 554                                          !err));
 555
 556                 if (kthread_should_stop())
 557                         break;
 558
 559                 /* dequeue the buffer and start capture */
 560                 spin_lock(&chan->start_lock);
 561                 if (list_empty(&chan->capture)) {
 562                         spin_unlock(&chan->start_lock);
 563                         continue;
 564                 }
 565
 566                 buf = list_first_entry(&chan->capture,
 567                                        struct tegra_channel_buffer, queue);
 568                 list_del_init(&buf->queue);
 569                 spin_unlock(&chan->start_lock);
 570
 571                 err = tegra_channel_capture_frame(chan, buf);
 572                 if (!err) {
 573                         retries = 0;
 574                         continue;
 575                 }
 576
 577                 if (retries++ > chan->syncpt_timeout_retry)
 578                         vb2_queue_error(&chan->queue);
 579                 else
 580                         err = 0;
 581         }
 582
 583         return 0;
 584 }
 585
 586 static int chan_capture_kthread_finish(void *data)
 587 {
 588         struct tegra_vi_channel *chan = data;
 589         struct tegra_channel_buffer *buf;
 590
 591         while (1) {
 592                 wait_event_interruptible(chan->done_wait,
 593                                          !list_empty(&chan->done) ||
 594                                          kthread_should_stop());
 595
 596                 /* dequeue buffers and finish capture */
 597                 buf = dequeue_buf_done(chan);
 598                 while (buf) {
 599                         tegra_channel_capture_done(chan, buf);
 600                         buf = dequeue_buf_done(chan);
 601                 }
 602
 603                 if (kthread_should_stop())
 604                         break;
 605         }
 606
 607         return 0;
 608 }
 609
 610 static int tegra210_vi_start_streaming(struct vb2_queue *vq, u32 count)
 611 {
 612         struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 613         struct media_pipeline *pipe = &chan->video.pipe;
 614         u32 val;
 615         u8 *portnos = chan->portnos;
 616         int ret, i;
 617
 618         tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 619
 620         /* clear syncpt errors */
 621         val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 622         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 623
 624         /*
 625          * Sync point FIFO full stalls the host interface.
 626          * Setting NO_STALL will drop INCR_SYNCPT methods when fifos are
 627          * full and the corresponding condition bits in INCR_SYNCPT_ERROR
 628          * register will be set.
 629          * This allows SW to process error recovery.
 630          */
 631         tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL,
 632                        VI_INCR_SYNCPT_NO_STALL);
 633
 634         /* start the pipeline */
 635         ret = video_device_pipeline_start(&chan->video, pipe);
 636         if (ret < 0)
 637                 goto error_pipeline_start;
 638
 639         /* clear csi errors and do capture setup for all ports in gang mode */
 640         for (i = 0; i < chan->numgangports; i++) {
 641                 val = vi_csi_read(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS);
 642                 vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS, val);
 643
 644                 tegra_channel_capture_setup(chan, portnos[i]);
 645         }
 646
 647         ret = tegra_channel_set_stream(chan, true);
 648         if (ret < 0)
 649                 goto error_set_stream;
 650
 651         chan->sequence = 0;
 652
 653         /* start kthreads to capture data to buffer and return them */
 654         chan->kthread_start_capture = kthread_run(chan_capture_kthread_start,
 655                                                   chan, "%s:0",
 656                                                   chan->video.name);
 657         if (IS_ERR(chan->kthread_start_capture)) {
 658                 ret = PTR_ERR(chan->kthread_start_capture);
 659                 chan->kthread_start_capture = NULL;
 660                 dev_err(&chan->video.dev,
 661                         "failed to run capture start kthread: %d\n", ret);
 662                 goto error_kthread_start;
 663         }
 664
 665         chan->kthread_finish_capture = kthread_run(chan_capture_kthread_finish,
 666                                                    chan, "%s:1",
 667                                                    chan->video.name);
 668         if (IS_ERR(chan->kthread_finish_capture)) {
 669                 ret = PTR_ERR(chan->kthread_finish_capture);
 670                 chan->kthread_finish_capture = NULL;
 671                 dev_err(&chan->video.dev,
 672                         "failed to run capture finish kthread: %d\n", ret);
 673                 goto error_kthread_done;
 674         }
 675
 676         return 0;
 677
 678 error_kthread_done:
 679         kthread_stop(chan->kthread_start_capture);
 680 error_kthread_start:
 681         tegra_channel_set_stream(chan, false);
 682 error_set_stream:
 683         video_device_pipeline_stop(&chan->video);
 684 error_pipeline_start:
 685         tegra_channel_release_buffers(chan, VB2_BUF_STATE_QUEUED);
 686         return ret;
 687 }
 688
 689 static void tegra210_vi_stop_streaming(struct vb2_queue *vq)
 690 {
 691         struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 692
 693         if (chan->kthread_start_capture) {
 694                 kthread_stop(chan->kthread_start_capture);
 695                 chan->kthread_start_capture = NULL;
 696         }
 697
 698         if (chan->kthread_finish_capture) {
 699                 kthread_stop(chan->kthread_finish_capture);
 700                 chan->kthread_finish_capture = NULL;
 701         }
 702
 703         tegra_channel_release_buffers(chan, VB2_BUF_STATE_ERROR);
 704         tegra_channel_set_stream(chan, false);
 705         video_device_pipeline_stop(&chan->video);
 706 }
 707
 708 /*
 709  * Tegra210 VI Pixel memory format enum.
 710  * These format enum value gets programmed into corresponding Tegra VI
 711  * channel register bits.
 712  */
 713 enum tegra210_image_format {
 714         TEGRA210_IMAGE_FORMAT_T_L8 = 16,
 715
 716         TEGRA210_IMAGE_FORMAT_T_R16_I = 32,
 717         TEGRA210_IMAGE_FORMAT_T_B5G6R5,
 718         TEGRA210_IMAGE_FORMAT_T_R5G6B5,
 719         TEGRA210_IMAGE_FORMAT_T_A1B5G5R5,
 720         TEGRA210_IMAGE_FORMAT_T_A1R5G5B5,
 721         TEGRA210_IMAGE_FORMAT_T_B5G5R5A1,
 722         TEGRA210_IMAGE_FORMAT_T_R5G5B5A1,
 723         TEGRA210_IMAGE_FORMAT_T_A4B4G4R4,
 724         TEGRA210_IMAGE_FORMAT_T_A4R4G4B4,
 725         TEGRA210_IMAGE_FORMAT_T_B4G4R4A4,
 726         TEGRA210_IMAGE_FORMAT_T_R4G4B4A4,
 727
 728         TEGRA210_IMAGE_FORMAT_T_A8B8G8R8 = 64,
 729         TEGRA210_IMAGE_FORMAT_T_A8R8G8B8,
 730         TEGRA210_IMAGE_FORMAT_T_B8G8R8A8,
 731         TEGRA210_IMAGE_FORMAT_T_R8G8B8A8,
 732         TEGRA210_IMAGE_FORMAT_T_A2B10G10R10,
 733         TEGRA210_IMAGE_FORMAT_T_A2R10G10B10,
 734         TEGRA210_IMAGE_FORMAT_T_B10G10R10A2,
 735         TEGRA210_IMAGE_FORMAT_T_R10G10B10A2,
 736
 737         TEGRA210_IMAGE_FORMAT_T_A8Y8U8V8 = 193,
 738         TEGRA210_IMAGE_FORMAT_T_V8U8Y8A8,
 739
 740         TEGRA210_IMAGE_FORMAT_T_A2Y10U10V10 = 197,
 741         TEGRA210_IMAGE_FORMAT_T_V10U10Y10A2,
 742         TEGRA210_IMAGE_FORMAT_T_Y8_U8__Y8_V8,
 743         TEGRA210_IMAGE_FORMAT_T_Y8_V8__Y8_U8,
 744         TEGRA210_IMAGE_FORMAT_T_U8_Y8__V8_Y8,
 745         TEGRA210_IMAGE_FORMAT_T_V8_Y8__U8_Y8,
 746
 747         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N444 = 224,
 748         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N444,
 749         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N444,
 750         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N422,
 751         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N422,
 752         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N422,
 753         TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N420,
 754         TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N420,
 755         TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N420,
 756         TEGRA210_IMAGE_FORMAT_T_X2LC10LB10LA10,
 757         TEGRA210_IMAGE_FORMAT_T_A2R6R6R6R6R6,
 758 };
 759
 760 #define TEGRA210_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP,        \
 761                            FORMAT, FOURCC)                              \
 762 {                                                                       \
 763         TEGRA_IMAGE_DT_##DATA_TYPE,                                     \
 764         BIT_WIDTH,                                                      \
 765         MEDIA_BUS_FMT_##MBUS_CODE,                                      \
 766         BPP,                                                            \
 767         TEGRA210_IMAGE_FORMAT_##FORMAT,                                 \
 768         V4L2_PIX_FMT_##FOURCC,                                          \
 769 }
 770
 771 /* Tegra210 supported video formats */
 772 static const struct tegra_video_format tegra210_video_formats[] = {
 773         /* RAW 8 */
 774         TEGRA210_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 1, T_L8, SRGGB8),
 775         TEGRA210_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 1, T_L8, SGRBG8),
 776         TEGRA210_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 1, T_L8, SGBRG8),
 777         TEGRA210_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 1, T_L8, SBGGR8),
 778         /* RAW 10 */
 779         TEGRA210_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, T_R16_I, SRGGB10),
 780         TEGRA210_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, T_R16_I, SGRBG10),
 781         TEGRA210_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, T_R16_I, SGBRG10),
 782         TEGRA210_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, T_R16_I, SBGGR10),
 783         /* RAW 12 */
 784         TEGRA210_VIDEO_FMT(RAW12, 12, SRGGB12_1X12, 2, T_R16_I, SRGGB12),
 785         TEGRA210_VIDEO_FMT(RAW12, 12, SGRBG12_1X12, 2, T_R16_I, SGRBG12),
 786         TEGRA210_VIDEO_FMT(RAW12, 12, SGBRG12_1X12, 2, T_R16_I, SGBRG12),
 787         TEGRA210_VIDEO_FMT(RAW12, 12, SBGGR12_1X12, 2, T_R16_I, SBGGR12),
 788         /* RGB888 */
 789         TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X24, 4, T_A8R8G8B8, XBGR32),
 790         TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X32_PADHI, 4, T_A8B8G8R8,
 791                            RGBX32),
 792         /* YUV422 */
 793         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, T_U8_Y8__V8_Y8, YVYU),
 794         TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, T_V8_Y8__U8_Y8, YUYV),
 795         TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, T_Y8_U8__Y8_V8, VYUY),
 796         TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, T_Y8_V8__Y8_U8, UYVY),
 797         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, T_Y8__V8U8_N422, NV16),
 798         TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, T_U8_Y8__V8_Y8, YVYU),
 799         TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, T_V8_Y8__U8_Y8, YUYV),
 800         TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, T_Y8_U8__Y8_V8, VYUY),
 801         TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, T_Y8_V8__Y8_U8, UYVY),
 802 };
 803
 804 /* Tegra210 VI operations */
 805 static const struct tegra_vi_ops tegra210_vi_ops = {
 806         .channel_host1x_syncpt_init = tegra210_channel_host1x_syncpt_init,
 807         .channel_host1x_syncpt_free = tegra210_channel_host1x_syncpt_free,
 808         .vi_fmt_align = tegra210_fmt_align,
 809         .vi_start_streaming = tegra210_vi_start_streaming,
 810         .vi_stop_streaming = tegra210_vi_stop_streaming,
 811 };
 812
 813 /* Tegra210 VI SoC data */
 814 const struct tegra_vi_soc tegra210_vi_soc = {
 815         .video_formats = tegra210_video_formats,
 816         .nformats = ARRAY_SIZE(tegra210_video_formats),
 817         .ops = &tegra210_vi_ops,
 818         .hw_revision = 3,
 819         .vi_max_channels = 6,
 820 #if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
 821         .default_video_format = &tegra210_video_formats[0],
 822         .vi_max_clk_hz = 499200000,
 823 #else
 824         .default_video_format = &tegra210_video_formats[4],
 825         .vi_max_clk_hz = 998400000,
 826 #endif
 827 };
 828
 829 /* Tegra210 CSI PHY registers accessors */
 830 static void csi_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 831                       u32 val)
 832 {
 833         void __iomem *csi_pp_base;
 834
 835         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 836
 837         writel_relaxed(val, csi_pp_base + addr);
 838 }
 839
 840 /* Tegra210 CSI Pixel parser registers accessors */
 841 static void pp_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 842 {
 843         void __iomem *csi_pp_base;
 844         unsigned int offset;
 845
 846         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 847         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 848
 849         writel_relaxed(val, csi_pp_base + offset + addr);
 850 }
 851
 852 static u32 pp_read(struct tegra_csi *csi, u8 portno, u32 addr)
 853 {
 854         void __iomem *csi_pp_base;
 855         unsigned int offset;
 856
 857         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 858         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 859
 860         return readl_relaxed(csi_pp_base + offset + addr);
 861 }
 862
 863 /* Tegra210 CSI CIL A/B port registers accessors */
 864 static void cil_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 865 {
 866         void __iomem *csi_cil_base;
 867         unsigned int offset;
 868
 869         csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 870                        TEGRA210_CSI_CIL_OFFSET;
 871         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 872
 873         writel_relaxed(val, csi_cil_base + offset + addr);
 874 }
 875
 876 static u32 cil_read(struct tegra_csi *csi, u8 portno, u32 addr)
 877 {
 878         void __iomem *csi_cil_base;
 879         unsigned int offset;
 880
 881         csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 882                        TEGRA210_CSI_CIL_OFFSET;
 883         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 884
 885         return readl_relaxed(csi_cil_base + offset + addr);
 886 }
 887
 888 /* Tegra210 CSI Test pattern generator registers accessor */
 889 static void tpg_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 890                       u32 val)
 891 {
 892         void __iomem *csi_pp_base;
 893         unsigned int offset;
 894
 895         csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 896         offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET +
 897                  TEGRA210_CSI_TPG_OFFSET;
 898
 899         writel_relaxed(val, csi_pp_base + offset + addr);
 900 }
 901
 902 /*
 903  * Tegra210 CSI operations
 904  */
 905 static void tegra210_csi_port_recover(struct tegra_csi_channel *csi_chan,
 906                                       u8 portno)
 907 {
 908         struct tegra_csi *csi = csi_chan->csi;
 909         u32 val;
 910
 911         /*
 912          * Recover CSI hardware in case of capture errors by issuing
 913          * software reset to CSICIL sensor, pixel parser, and clear errors
 914          * to have clean capture on  next streaming.
 915          */
 916         val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
 917         dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
 918
 919         val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
 920         dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
 921
 922         val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
 923         dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
 924
 925         if (csi_chan->numlanes == 4) {
 926                 /* reset CSI CIL sensor */
 927                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 928                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 929                 /*
 930                  * SW_STATUS_RESET resets all status bits of PPA, PPB, CILA,
 931                  * CILB status registers and debug counters.
 932                  * So, SW_STATUS_RESET can be used only when CSI brick is in
 933                  * x4 mode.
 934                  */
 935                 csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x1);
 936
 937                 /* sleep for 20 clock cycles to drain the FIFO */
 938                 usleep_range(10, 20);
 939
 940                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 941                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 942                 csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x0);
 943         } else {
 944                 /* reset CSICIL sensor */
 945                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 946                 usleep_range(10, 20);
 947                 cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 948
 949                 /* clear the errors */
 950                 pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS,
 951                          0xffffffff);
 952                 cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 953                 cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 954         }
 955 }
 956
 957 static void tegra210_csi_error_recover(struct tegra_csi_channel *csi_chan)
 958 {
 959         u8 *portnos = csi_chan->csi_port_nums;
 960         int i;
 961
 962         for (i = 0; i < csi_chan->numgangports; i++)
 963                 tegra210_csi_port_recover(csi_chan, portnos[i]);
 964 }
 965
 966 static int
 967 tegra210_csi_port_start_streaming(struct tegra_csi_channel *csi_chan,
 968                                   u8 portno)
 969 {
 970         struct tegra_csi *csi = csi_chan->csi;
 971         u8 clk_settle_time = 0;
 972         u8 ths_settle_time = 10;
 973         u32 val;
 974
 975         if (!csi_chan->pg_mode)
 976                 tegra_csi_calc_settle_time(csi_chan, portno, &clk_settle_time,
 977                                            &ths_settle_time);
 978
 979         csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
 980
 981         /* clean up status */
 982         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, 0xffffffff);
 983         cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 984         cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 985         cil_write(csi, portno, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
 986
 987         /* CIL PHY registers setup */
 988         cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
 989         cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL,
 990                   FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
 991                   FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
 992
 993         /*
 994          * The CSI unit provides for connection of up to six cameras in
 995          * the system and is organized as three identical instances of
 996          * two MIPI support blocks, each with a separate 4-lane
 997          * interface that can be configured as a single camera with 4
 998          * lanes or as a dual camera with 2 lanes available for each
 999          * camera.
1000          */
1001         if (csi_chan->numlanes == 4) {
1002                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_STATUS, 0xffffffff);
1003                 cil_write(csi, portno + 1, TEGRA_CSI_CILX_STATUS, 0xffffffff);
1004                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
1005
1006                 cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0,
1007                           BRICK_CLOCK_A_4X);
1008                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
1009                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
1010                 cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL,
1011                           FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
1012                           FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
1013                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
1014                           CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
1015         } else {
1016                 val = ((portno & 1) == PORT_A) ?
1017                       CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_NOP :
1018                       CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_NOP;
1019                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
1020         }
1021
1022         /* CSI pixel parser registers setup */
1023         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1024                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1025                  CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_RST);
1026         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0x0);
1027         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL0,
1028                  CSI_PP_PACKET_HEADER_SENT |
1029                  CSI_PP_DATA_IDENTIFIER_ENABLE |
1030                  CSI_PP_WORD_COUNT_SELECT_HEADER |
1031                  CSI_PP_CRC_CHECK_ENABLE |  CSI_PP_WC_CHECK |
1032                  CSI_PP_OUTPUT_FORMAT_STORE | CSI_PPA_PAD_LINE_NOPAD |
1033                  CSI_PP_HEADER_EC_DISABLE | CSI_PPA_PAD_FRAME_NOPAD |
1034                  (portno & 1));
1035         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL1,
1036                  (0x1 << CSI_PP_TOP_FIELD_FRAME_OFFSET) |
1037                  (0x1 << CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET));
1038         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_GAP,
1039                  0x14 << PP_FRAME_MIN_GAP_OFFSET);
1040         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME, 0x0);
1041         pp_write(csi, portno, TEGRA_CSI_INPUT_STREAM_CONTROL,
1042                  (0x3f << CSI_SKIP_PACKET_THRESHOLD_OFFSET) |
1043                  (csi_chan->numlanes - 1));
1044
1045         /* TPG setup */
1046         if (csi_chan->pg_mode) {
1047                 tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
1048                           ((csi_chan->pg_mode - 1) << PG_MODE_OFFSET) |
1049                           PG_ENABLE);
1050                 tpg_write(csi, portno, TEGRA_CSI_PG_BLANK,
1051                           csi_chan->v_blank << PG_VBLANK_OFFSET |
1052                           csi_chan->h_blank);
1053                 tpg_write(csi, portno, TEGRA_CSI_PG_PHASE, 0x0);
1054                 tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ,
1055                           (0x10 << PG_RED_VERT_INIT_FREQ_OFFSET) |
1056                           (0x10 << PG_RED_HOR_INIT_FREQ_OFFSET));
1057                 tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ_RATE, 0x0);
1058                 tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ,
1059                           (0x10 << PG_GREEN_VERT_INIT_FREQ_OFFSET) |
1060                           (0x10 << PG_GREEN_HOR_INIT_FREQ_OFFSET));
1061                 tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ_RATE, 0x0);
1062                 tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ,
1063                           (0x10 << PG_BLUE_VERT_INIT_FREQ_OFFSET) |
1064                           (0x10 << PG_BLUE_HOR_INIT_FREQ_OFFSET));
1065                 tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ_RATE, 0x0);
1066         }
1067
1068         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1069                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1070                  CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_ENABLE);
1071
1072         return 0;
1073 }
1074
1075 static void
1076 tegra210_csi_port_stop_streaming(struct tegra_csi_channel *csi_chan, u8 portno)
1077 {
1078         struct tegra_csi *csi = csi_chan->csi;
1079         u32 val;
1080
1081         val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
1082
1083         dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
1084         pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, val);
1085
1086         val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
1087         dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
1088         cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, val);
1089
1090         val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
1091         dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
1092         cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, val);
1093
1094         pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1095                  (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1096                  CSI_PP_DISABLE);
1097
1098         if (csi_chan->pg_mode) {
1099                 tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
1100                           PG_DISABLE);
1101                 return;
1102         }
1103
1104         if (csi_chan->numlanes == 4) {
1105                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
1106                           CSI_A_PHY_CIL_DISABLE |
1107                           CSI_B_PHY_CIL_DISABLE);
1108         } else {
1109                 val = ((portno & 1) == PORT_A) ?
1110                       CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
1111                       CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
1112                 csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
1113         }
1114 }
1115
1116 static int tegra210_csi_start_streaming(struct tegra_csi_channel *csi_chan)
1117 {
1118         u8 *portnos = csi_chan->csi_port_nums;
1119         int ret, i;
1120
1121         for (i = 0; i < csi_chan->numgangports; i++) {
1122                 ret = tegra210_csi_port_start_streaming(csi_chan, portnos[i]);
1123                 if (ret)
1124                         goto stream_start_fail;
1125         }
1126
1127         return 0;
1128
1129 stream_start_fail:
1130         for (i = i - 1; i >= 0; i--)
1131                 tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1132
1133         return ret;
1134 }
1135
1136 static void tegra210_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
1137 {
1138         u8 *portnos = csi_chan->csi_port_nums;
1139         int i;
1140
1141         for (i = 0; i < csi_chan->numgangports; i++)
1142                 tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1143 }
1144
1145 /*
1146  * Tegra210 CSI TPG frame rate table with horizontal and vertical
1147  * blanking intervals for corresponding format and resolution.
1148  * Blanking intervals are tuned values from design team for max TPG
1149  * clock rate.
1150  */
1151 static const struct tpg_framerate tegra210_tpg_frmrate_table[] = {
1152         {
1153                 .frmsize = { 1280, 720 },
1154                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1155                 .framerate = 120,
1156                 .h_blank = 512,
1157                 .v_blank = 8,
1158         },
1159         {
1160                 .frmsize = { 1920, 1080 },
1161                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1162                 .framerate = 60,
1163                 .h_blank = 512,
1164                 .v_blank = 8,
1165         },
1166         {
1167                 .frmsize = { 3840, 2160 },
1168                 .code = MEDIA_BUS_FMT_SRGGB10_1X10,
1169                 .framerate = 20,
1170                 .h_blank = 8,
1171                 .v_blank = 8,
1172         },
1173         {
1174                 .frmsize = { 1280, 720 },
1175                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1176                 .framerate = 60,
1177                 .h_blank = 512,
1178                 .v_blank = 8,
1179         },
1180         {
1181                 .frmsize = { 1920, 1080 },
1182                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1183                 .framerate = 30,
1184                 .h_blank = 512,
1185                 .v_blank = 8,
1186         },
1187         {
1188                 .frmsize = { 3840, 2160 },
1189                 .code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1190                 .framerate = 8,
1191                 .h_blank = 8,
1192                 .v_blank = 8,
1193         },
1194 };
1195
1196 static const char * const tegra210_csi_cil_clks[] = {
1197         "csi",
1198         "cilab",
1199         "cilcd",
1200         "cile",
1201 #if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
1202         "csi_tpg",
1203 #endif
1204 };
1205
1206 /* Tegra210 CSI operations */
1207 static const struct tegra_csi_ops tegra210_csi_ops = {
1208         .csi_start_streaming = tegra210_csi_start_streaming,
1209         .csi_stop_streaming = tegra210_csi_stop_streaming,
1210         .csi_err_recover = tegra210_csi_error_recover,
1211 };
1212
1213 /* Tegra210 CSI SoC data */
1214 const struct tegra_csi_soc tegra210_csi_soc = {
1215         .ops = &tegra210_csi_ops,
1216         .csi_max_channels = 6,
1217         .clk_names = tegra210_csi_cil_clks,
1218         .num_clks = ARRAY_SIZE(tegra210_csi_cil_clks),
1219         .tpg_frmrate_table = tegra210_tpg_frmrate_table,
1220         .tpg_frmrate_table_size = ARRAY_SIZE(tegra210_tpg_frmrate_table),
1221 };