drivers/tty/serial/msm_serial.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for msm7k serial device and console
   4  *
   5  * Copyright (C) 2007 Google, Inc.
   6  * Author: Robert Love <rlove@google.com>
   7  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/atomic.h>
  12 #include <linux/dma/qcom_adm.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/module.h>
  16 #include <linux/io.h>
  17 #include <linux/ioport.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/init.h>
  20 #include <linux/console.h>
  21 #include <linux/tty.h>
  22 #include <linux/tty_flip.h>
  23 #include <linux/serial_core.h>
  24 #include <linux/slab.h>
  25 #include <linux/clk.h>
  26 #include <linux/platform_device.h>
  27 #include <linux/delay.h>
  28 #include <linux/of.h>
  29 #include <linux/of_device.h>
  30 #include <linux/wait.h>
  31
  32 #define UART_MR1                        0x0000
  33
  34 #define UART_MR1_AUTO_RFR_LEVEL0        0x3F
  35 #define UART_MR1_AUTO_RFR_LEVEL1        0x3FF00
  36 #define UART_DM_MR1_AUTO_RFR_LEVEL1     0xFFFFFF00
  37 #define UART_MR1_RX_RDY_CTL             BIT(7)
  38 #define UART_MR1_CTS_CTL                BIT(6)
  39
  40 #define UART_MR2                        0x0004
  41 #define UART_MR2_ERROR_MODE             BIT(6)
  42 #define UART_MR2_BITS_PER_CHAR          0x30
  43 #define UART_MR2_BITS_PER_CHAR_5        (0x0 << 4)
  44 #define UART_MR2_BITS_PER_CHAR_6        (0x1 << 4)
  45 #define UART_MR2_BITS_PER_CHAR_7        (0x2 << 4)
  46 #define UART_MR2_BITS_PER_CHAR_8        (0x3 << 4)
  47 #define UART_MR2_STOP_BIT_LEN_ONE       (0x1 << 2)
  48 #define UART_MR2_STOP_BIT_LEN_TWO       (0x3 << 2)
  49 #define UART_MR2_PARITY_MODE_NONE       0x0
  50 #define UART_MR2_PARITY_MODE_ODD        0x1
  51 #define UART_MR2_PARITY_MODE_EVEN       0x2
  52 #define UART_MR2_PARITY_MODE_SPACE      0x3
  53 #define UART_MR2_PARITY_MODE            0x3
  54
  55 #define UART_CSR                        0x0008
  56
  57 #define UART_TF                         0x000C
  58 #define UARTDM_TF                       0x0070
  59
  60 #define UART_CR                         0x0010
  61 #define UART_CR_CMD_NULL                (0 << 4)
  62 #define UART_CR_CMD_RESET_RX            (1 << 4)
  63 #define UART_CR_CMD_RESET_TX            (2 << 4)
  64 #define UART_CR_CMD_RESET_ERR           (3 << 4)
  65 #define UART_CR_CMD_RESET_BREAK_INT     (4 << 4)
  66 #define UART_CR_CMD_START_BREAK         (5 << 4)
  67 #define UART_CR_CMD_STOP_BREAK          (6 << 4)
  68 #define UART_CR_CMD_RESET_CTS           (7 << 4)
  69 #define UART_CR_CMD_RESET_STALE_INT     (8 << 4)
  70 #define UART_CR_CMD_PACKET_MODE         (9 << 4)
  71 #define UART_CR_CMD_MODE_RESET          (12 << 4)
  72 #define UART_CR_CMD_SET_RFR             (13 << 4)
  73 #define UART_CR_CMD_RESET_RFR           (14 << 4)
  74 #define UART_CR_CMD_PROTECTION_EN       (16 << 4)
  75 #define UART_CR_CMD_STALE_EVENT_DISABLE (6 << 8)
  76 #define UART_CR_CMD_STALE_EVENT_ENABLE  (80 << 4)
  77 #define UART_CR_CMD_FORCE_STALE         (4 << 8)
  78 #define UART_CR_CMD_RESET_TX_READY      (3 << 8)
  79 #define UART_CR_TX_DISABLE              BIT(3)
  80 #define UART_CR_TX_ENABLE               BIT(2)
  81 #define UART_CR_RX_DISABLE              BIT(1)
  82 #define UART_CR_RX_ENABLE               BIT(0)
  83 #define UART_CR_CMD_RESET_RXBREAK_START ((1 << 11) | (2 << 4))
  84
  85 #define UART_IMR                        0x0014
  86 #define UART_IMR_TXLEV                  BIT(0)
  87 #define UART_IMR_RXSTALE                BIT(3)
  88 #define UART_IMR_RXLEV                  BIT(4)
  89 #define UART_IMR_DELTA_CTS              BIT(5)
  90 #define UART_IMR_CURRENT_CTS            BIT(6)
  91 #define UART_IMR_RXBREAK_START          BIT(10)
  92
  93 #define UART_IPR_RXSTALE_LAST           0x20
  94 #define UART_IPR_STALE_LSB              0x1F
  95 #define UART_IPR_STALE_TIMEOUT_MSB      0x3FF80
  96 #define UART_DM_IPR_STALE_TIMEOUT_MSB   0xFFFFFF80
  97
  98 #define UART_IPR                        0x0018
  99 #define UART_TFWR                       0x001C
 100 #define UART_RFWR                       0x0020
 101 #define UART_HCR                        0x0024
 102
 103 #define UART_MREG                       0x0028
 104 #define UART_NREG                       0x002C
 105 #define UART_DREG                       0x0030
 106 #define UART_MNDREG                     0x0034
 107 #define UART_IRDA                       0x0038
 108 #define UART_MISR_MODE                  0x0040
 109 #define UART_MISR_RESET                 0x0044
 110 #define UART_MISR_EXPORT                0x0048
 111 #define UART_MISR_VAL                   0x004C
 112 #define UART_TEST_CTRL                  0x0050
 113
 114 #define UART_SR                         0x0008
 115 #define UART_SR_HUNT_CHAR               BIT(7)
 116 #define UART_SR_RX_BREAK                BIT(6)
 117 #define UART_SR_PAR_FRAME_ERR           BIT(5)
 118 #define UART_SR_OVERRUN                 BIT(4)
 119 #define UART_SR_TX_EMPTY                BIT(3)
 120 #define UART_SR_TX_READY                BIT(2)
 121 #define UART_SR_RX_FULL                 BIT(1)
 122 #define UART_SR_RX_READY                BIT(0)
 123
 124 #define UART_RF                         0x000C
 125 #define UARTDM_RF                       0x0070
 126 #define UART_MISR                       0x0010
 127 #define UART_ISR                        0x0014
 128 #define UART_ISR_TX_READY               BIT(7)
 129
 130 #define UARTDM_RXFS                     0x50
 131 #define UARTDM_RXFS_BUF_SHIFT           0x7
 132 #define UARTDM_RXFS_BUF_MASK            0x7
 133
 134 #define UARTDM_DMEN                     0x3C
 135 #define UARTDM_DMEN_RX_SC_ENABLE        BIT(5)
 136 #define UARTDM_DMEN_TX_SC_ENABLE        BIT(4)
 137
 138 #define UARTDM_DMEN_TX_BAM_ENABLE       BIT(2)  /* UARTDM_1P4 */
 139 #define UARTDM_DMEN_TX_DM_ENABLE        BIT(0)  /* < UARTDM_1P4 */
 140
 141 #define UARTDM_DMEN_RX_BAM_ENABLE       BIT(3)  /* UARTDM_1P4 */
 142 #define UARTDM_DMEN_RX_DM_ENABLE        BIT(1)  /* < UARTDM_1P4 */
 143
 144 #define UARTDM_DMRX                     0x34
 145 #define UARTDM_NCF_TX                   0x40
 146 #define UARTDM_RX_TOTAL_SNAP            0x38
 147
 148 #define UARTDM_BURST_SIZE               16   /* in bytes */
 149 #define UARTDM_TX_AIGN(x)               ((x) & ~0x3) /* valid for > 1p3 */
 150 #define UARTDM_TX_MAX                   256   /* in bytes, valid for <= 1p3 */
 151 #define UARTDM_RX_SIZE                  (UART_XMIT_SIZE / 4)
 152
 153 enum {
 154         UARTDM_1P1 = 1,
 155         UARTDM_1P2,
 156         UARTDM_1P3,
 157         UARTDM_1P4,
 158 };
 159
 160 struct msm_dma {
 161         struct dma_chan         *chan;
 162         enum dma_data_direction dir;
 163         dma_addr_t              phys;
 164         unsigned char           *virt;
 165         dma_cookie_t            cookie;
 166         u32                     enable_bit;
 167         unsigned int            count;
 168         struct dma_async_tx_descriptor  *desc;
 169 };
 170
 171 struct msm_port {
 172         struct uart_port        uart;
 173         char                    name[16];
 174         struct clk              *clk;
 175         struct clk              *pclk;
 176         unsigned int            imr;
 177         int                     is_uartdm;
 178         unsigned int            old_snap_state;
 179         bool                    break_detected;
 180         struct msm_dma          tx_dma;
 181         struct msm_dma          rx_dma;
 182 };
 183
 184 #define UART_TO_MSM(uart_port)  container_of(uart_port, struct msm_port, uart)
 185
 186 static
 187 void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 188 {
 189         writel_relaxed(val, port->membase + off);
 190 }
 191
 192 static
 193 unsigned int msm_read(struct uart_port *port, unsigned int off)
 194 {
 195         return readl_relaxed(port->membase + off);
 196 }
 197
 198 /*
 199  * Setup the MND registers to use the TCXO clock.
 200  */
 201 static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 202 {
 203         msm_write(port, 0x06, UART_MREG);
 204         msm_write(port, 0xF1, UART_NREG);
 205         msm_write(port, 0x0F, UART_DREG);
 206         msm_write(port, 0x1A, UART_MNDREG);
 207         port->uartclk = 1843200;
 208 }
 209
 210 /*
 211  * Setup the MND registers to use the TCXO clock divided by 4.
 212  */
 213 static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 214 {
 215         msm_write(port, 0x18, UART_MREG);
 216         msm_write(port, 0xF6, UART_NREG);
 217         msm_write(port, 0x0F, UART_DREG);
 218         msm_write(port, 0x0A, UART_MNDREG);
 219         port->uartclk = 1843200;
 220 }
 221
 222 static void msm_serial_set_mnd_regs(struct uart_port *port)
 223 {
 224         struct msm_port *msm_port = UART_TO_MSM(port);
 225
 226         /*
 227          * These registers don't exist so we change the clk input rate
 228          * on uartdm hardware instead
 229          */
 230         if (msm_port->is_uartdm)
 231                 return;
 232
 233         if (port->uartclk == 19200000)
 234                 msm_serial_set_mnd_regs_tcxo(port);
 235         else if (port->uartclk == 4800000)
 236                 msm_serial_set_mnd_regs_tcxoby4(port);
 237 }
 238
 239 static void msm_handle_tx(struct uart_port *port);
 240 static void msm_start_rx_dma(struct msm_port *msm_port);
 241
 242 static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 243 {
 244         struct device *dev = port->dev;
 245         unsigned int mapped;
 246         u32 val;
 247
 248         mapped = dma->count;
 249         dma->count = 0;
 250
 251         dmaengine_terminate_all(dma->chan);
 252
 253         /*
 254          * DMA Stall happens if enqueue and flush command happens concurrently.
 255          * For example before changing the baud rate/protocol configuration and
 256          * sending flush command to ADM, disable the channel of UARTDM.
 257          * Note: should not reset the receiver here immediately as it is not
 258          * suggested to do disable/reset or reset/disable at the same time.
 259          */
 260         val = msm_read(port, UARTDM_DMEN);
 261         val &= ~dma->enable_bit;
 262         msm_write(port, val, UARTDM_DMEN);
 263
 264         if (mapped)
 265                 dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 266 }
 267
 268 static void msm_release_dma(struct msm_port *msm_port)
 269 {
 270         struct msm_dma *dma;
 271
 272         dma = &msm_port->tx_dma;
 273         if (dma->chan) {
 274                 msm_stop_dma(&msm_port->uart, dma);
 275                 dma_release_channel(dma->chan);
 276         }
 277
 278         memset(dma, 0, sizeof(*dma));
 279
 280         dma = &msm_port->rx_dma;
 281         if (dma->chan) {
 282                 msm_stop_dma(&msm_port->uart, dma);
 283                 dma_release_channel(dma->chan);
 284                 kfree(dma->virt);
 285         }
 286
 287         memset(dma, 0, sizeof(*dma));
 288 }
 289
 290 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 291 {
 292         struct device *dev = msm_port->uart.dev;
 293         struct dma_slave_config conf;
 294         struct qcom_adm_peripheral_config periph_conf = {};
 295         struct msm_dma *dma;
 296         u32 crci = 0;
 297         int ret;
 298
 299         dma = &msm_port->tx_dma;
 300
 301         /* allocate DMA resources, if available */
 302         dma->chan = dma_request_chan(dev, "tx");
 303         if (IS_ERR(dma->chan))
 304                 goto no_tx;
 305
 306         of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 307
 308         memset(&conf, 0, sizeof(conf));
 309         conf.direction = DMA_MEM_TO_DEV;
 310         conf.device_fc = true;
 311         conf.dst_addr = base + UARTDM_TF;
 312         conf.dst_maxburst = UARTDM_BURST_SIZE;
 313         if (crci) {
 314                 conf.peripheral_config = &periph_conf;
 315                 conf.peripheral_size = sizeof(periph_conf);
 316                 periph_conf.crci = crci;
 317         }
 318
 319         ret = dmaengine_slave_config(dma->chan, &conf);
 320         if (ret)
 321                 goto rel_tx;
 322
 323         dma->dir = DMA_TO_DEVICE;
 324
 325         if (msm_port->is_uartdm < UARTDM_1P4)
 326                 dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 327         else
 328                 dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 329
 330         return;
 331
 332 rel_tx:
 333         dma_release_channel(dma->chan);
 334 no_tx:
 335         memset(dma, 0, sizeof(*dma));
 336 }
 337
 338 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 339 {
 340         struct device *dev = msm_port->uart.dev;
 341         struct dma_slave_config conf;
 342         struct qcom_adm_peripheral_config periph_conf = {};
 343         struct msm_dma *dma;
 344         u32 crci = 0;
 345         int ret;
 346
 347         dma = &msm_port->rx_dma;
 348
 349         /* allocate DMA resources, if available */
 350         dma->chan = dma_request_chan(dev, "rx");
 351         if (IS_ERR(dma->chan))
 352                 goto no_rx;
 353
 354         of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 355
 356         dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 357         if (!dma->virt)
 358                 goto rel_rx;
 359
 360         memset(&conf, 0, sizeof(conf));
 361         conf.direction = DMA_DEV_TO_MEM;
 362         conf.device_fc = true;
 363         conf.src_addr = base + UARTDM_RF;
 364         conf.src_maxburst = UARTDM_BURST_SIZE;
 365         if (crci) {
 366                 conf.peripheral_config = &periph_conf;
 367                 conf.peripheral_size = sizeof(periph_conf);
 368                 periph_conf.crci = crci;
 369         }
 370
 371         ret = dmaengine_slave_config(dma->chan, &conf);
 372         if (ret)
 373                 goto err;
 374
 375         dma->dir = DMA_FROM_DEVICE;
 376
 377         if (msm_port->is_uartdm < UARTDM_1P4)
 378                 dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 379         else
 380                 dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 381
 382         return;
 383 err:
 384         kfree(dma->virt);
 385 rel_rx:
 386         dma_release_channel(dma->chan);
 387 no_rx:
 388         memset(dma, 0, sizeof(*dma));
 389 }
 390
 391 static inline void msm_wait_for_xmitr(struct uart_port *port)
 392 {
 393         unsigned int timeout = 500000;
 394
 395         while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 396                 if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 397                         break;
 398                 udelay(1);
 399                 if (!timeout--)
 400                         break;
 401         }
 402         msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 403 }
 404
 405 static void msm_stop_tx(struct uart_port *port)
 406 {
 407         struct msm_port *msm_port = UART_TO_MSM(port);
 408
 409         msm_port->imr &= ~UART_IMR_TXLEV;
 410         msm_write(port, msm_port->imr, UART_IMR);
 411 }
 412
 413 static void msm_start_tx(struct uart_port *port)
 414 {
 415         struct msm_port *msm_port = UART_TO_MSM(port);
 416         struct msm_dma *dma = &msm_port->tx_dma;
 417
 418         /* Already started in DMA mode */
 419         if (dma->count)
 420                 return;
 421
 422         msm_port->imr |= UART_IMR_TXLEV;
 423         msm_write(port, msm_port->imr, UART_IMR);
 424 }
 425
 426 static void msm_reset_dm_count(struct uart_port *port, int count)
 427 {
 428         msm_wait_for_xmitr(port);
 429         msm_write(port, count, UARTDM_NCF_TX);
 430         msm_read(port, UARTDM_NCF_TX);
 431 }
 432
 433 static void msm_complete_tx_dma(void *args)
 434 {
 435         struct msm_port *msm_port = args;
 436         struct uart_port *port = &msm_port->uart;
 437         struct circ_buf *xmit = &port->state->xmit;
 438         struct msm_dma *dma = &msm_port->tx_dma;
 439         struct dma_tx_state state;
 440         unsigned long flags;
 441         unsigned int count;
 442         u32 val;
 443
 444         spin_lock_irqsave(&port->lock, flags);
 445
 446         /* Already stopped */
 447         if (!dma->count)
 448                 goto done;
 449
 450         dmaengine_tx_status(dma->chan, dma->cookie, &state);
 451
 452         dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 453
 454         val = msm_read(port, UARTDM_DMEN);
 455         val &= ~dma->enable_bit;
 456         msm_write(port, val, UARTDM_DMEN);
 457
 458         if (msm_port->is_uartdm > UARTDM_1P3) {
 459                 msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 460                 msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 461         }
 462
 463         count = dma->count - state.residue;
 464         port->icount.tx += count;
 465         dma->count = 0;
 466
 467         xmit->tail += count;
 468         xmit->tail &= UART_XMIT_SIZE - 1;
 469
 470         /* Restore "Tx FIFO below watermark" interrupt */
 471         msm_port->imr |= UART_IMR_TXLEV;
 472         msm_write(port, msm_port->imr, UART_IMR);
 473
 474         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 475                 uart_write_wakeup(port);
 476
 477         msm_handle_tx(port);
 478 done:
 479         spin_unlock_irqrestore(&port->lock, flags);
 480 }
 481
 482 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 483 {
 484         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 485         struct uart_port *port = &msm_port->uart;
 486         struct msm_dma *dma = &msm_port->tx_dma;
 487         void *cpu_addr;
 488         int ret;
 489         u32 val;
 490
 491         cpu_addr = &xmit->buf[xmit->tail];
 492
 493         dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 494         ret = dma_mapping_error(port->dev, dma->phys);
 495         if (ret)
 496                 return ret;
 497
 498         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 499                                                 count, DMA_MEM_TO_DEV,
 500                                                 DMA_PREP_INTERRUPT |
 501                                                 DMA_PREP_FENCE);
 502         if (!dma->desc) {
 503                 ret = -EIO;
 504                 goto unmap;
 505         }
 506
 507         dma->desc->callback = msm_complete_tx_dma;
 508         dma->desc->callback_param = msm_port;
 509
 510         dma->cookie = dmaengine_submit(dma->desc);
 511         ret = dma_submit_error(dma->cookie);
 512         if (ret)
 513                 goto unmap;
 514
 515         /*
 516          * Using DMA complete for Tx FIFO reload, no need for
 517          * "Tx FIFO below watermark" one, disable it
 518          */
 519         msm_port->imr &= ~UART_IMR_TXLEV;
 520         msm_write(port, msm_port->imr, UART_IMR);
 521
 522         dma->count = count;
 523
 524         val = msm_read(port, UARTDM_DMEN);
 525         val |= dma->enable_bit;
 526
 527         if (msm_port->is_uartdm < UARTDM_1P4)
 528                 msm_write(port, val, UARTDM_DMEN);
 529
 530         msm_reset_dm_count(port, count);
 531
 532         if (msm_port->is_uartdm > UARTDM_1P3)
 533                 msm_write(port, val, UARTDM_DMEN);
 534
 535         dma_async_issue_pending(dma->chan);
 536         return 0;
 537 unmap:
 538         dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 539         return ret;
 540 }
 541
 542 static void msm_complete_rx_dma(void *args)
 543 {
 544         struct msm_port *msm_port = args;
 545         struct uart_port *port = &msm_port->uart;
 546         struct tty_port *tport = &port->state->port;
 547         struct msm_dma *dma = &msm_port->rx_dma;
 548         int count = 0, i, sysrq;
 549         unsigned long flags;
 550         u32 val;
 551
 552         spin_lock_irqsave(&port->lock, flags);
 553
 554         /* Already stopped */
 555         if (!dma->count)
 556                 goto done;
 557
 558         val = msm_read(port, UARTDM_DMEN);
 559         val &= ~dma->enable_bit;
 560         msm_write(port, val, UARTDM_DMEN);
 561
 562         if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 563                 port->icount.overrun++;
 564                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 565                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 566         }
 567
 568         count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 569
 570         port->icount.rx += count;
 571
 572         dma->count = 0;
 573
 574         dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 575
 576         for (i = 0; i < count; i++) {
 577                 char flag = TTY_NORMAL;
 578
 579                 if (msm_port->break_detected && dma->virt[i] == 0) {
 580                         port->icount.brk++;
 581                         flag = TTY_BREAK;
 582                         msm_port->break_detected = false;
 583                         if (uart_handle_break(port))
 584                                 continue;
 585                 }
 586
 587                 if (!(port->read_status_mask & UART_SR_RX_BREAK))
 588                         flag = TTY_NORMAL;
 589
 590                 spin_unlock_irqrestore(&port->lock, flags);
 591                 sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 592                 spin_lock_irqsave(&port->lock, flags);
 593                 if (!sysrq)
 594                         tty_insert_flip_char(tport, dma->virt[i], flag);
 595         }
 596
 597         msm_start_rx_dma(msm_port);
 598 done:
 599         spin_unlock_irqrestore(&port->lock, flags);
 600
 601         if (count)
 602                 tty_flip_buffer_push(tport);
 603 }
 604
 605 static void msm_start_rx_dma(struct msm_port *msm_port)
 606 {
 607         struct msm_dma *dma = &msm_port->rx_dma;
 608         struct uart_port *uart = &msm_port->uart;
 609         u32 val;
 610         int ret;
 611
 612         if (IS_ENABLED(CONFIG_CONSOLE_POLL))
 613                 return;
 614
 615         if (!dma->chan)
 616                 return;
 617
 618         dma->phys = dma_map_single(uart->dev, dma->virt,
 619                                    UARTDM_RX_SIZE, dma->dir);
 620         ret = dma_mapping_error(uart->dev, dma->phys);
 621         if (ret)
 622                 goto sw_mode;
 623
 624         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 625                                                 UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 626                                                 DMA_PREP_INTERRUPT);
 627         if (!dma->desc)
 628                 goto unmap;
 629
 630         dma->desc->callback = msm_complete_rx_dma;
 631         dma->desc->callback_param = msm_port;
 632
 633         dma->cookie = dmaengine_submit(dma->desc);
 634         ret = dma_submit_error(dma->cookie);
 635         if (ret)
 636                 goto unmap;
 637         /*
 638          * Using DMA for FIFO off-load, no need for "Rx FIFO over
 639          * watermark" or "stale" interrupts, disable them
 640          */
 641         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 642
 643         /*
 644          * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 645          * we need RXSTALE to flush input DMA fifo to memory
 646          */
 647         if (msm_port->is_uartdm < UARTDM_1P4)
 648                 msm_port->imr |= UART_IMR_RXSTALE;
 649
 650         msm_write(uart, msm_port->imr, UART_IMR);
 651
 652         dma->count = UARTDM_RX_SIZE;
 653
 654         dma_async_issue_pending(dma->chan);
 655
 656         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 657         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 658
 659         val = msm_read(uart, UARTDM_DMEN);
 660         val |= dma->enable_bit;
 661
 662         if (msm_port->is_uartdm < UARTDM_1P4)
 663                 msm_write(uart, val, UARTDM_DMEN);
 664
 665         msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 666
 667         if (msm_port->is_uartdm > UARTDM_1P3)
 668                 msm_write(uart, val, UARTDM_DMEN);
 669
 670         return;
 671 unmap:
 672         dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 673
 674 sw_mode:
 675         /*
 676          * Switch from DMA to SW/FIFO mode. After clearing Rx BAM (UARTDM_DMEN),
 677          * receiver must be reset.
 678          */
 679         msm_write(uart, UART_CR_CMD_RESET_RX, UART_CR);
 680         msm_write(uart, UART_CR_RX_ENABLE, UART_CR);
 681
 682         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 683         msm_write(uart, 0xFFFFFF, UARTDM_DMRX);
 684         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 685
 686         /* Re-enable RX interrupts */
 687         msm_port->imr |= (UART_IMR_RXLEV | UART_IMR_RXSTALE);
 688         msm_write(uart, msm_port->imr, UART_IMR);
 689 }
 690
 691 static void msm_stop_rx(struct uart_port *port)
 692 {
 693         struct msm_port *msm_port = UART_TO_MSM(port);
 694         struct msm_dma *dma = &msm_port->rx_dma;
 695
 696         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 697         msm_write(port, msm_port->imr, UART_IMR);
 698
 699         if (dma->chan)
 700                 msm_stop_dma(port, dma);
 701 }
 702
 703 static void msm_enable_ms(struct uart_port *port)
 704 {
 705         struct msm_port *msm_port = UART_TO_MSM(port);
 706
 707         msm_port->imr |= UART_IMR_DELTA_CTS;
 708         msm_write(port, msm_port->imr, UART_IMR);
 709 }
 710
 711 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 712         __must_hold(&port->lock)
 713 {
 714         struct tty_port *tport = &port->state->port;
 715         unsigned int sr;
 716         int count = 0;
 717         struct msm_port *msm_port = UART_TO_MSM(port);
 718
 719         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 720                 port->icount.overrun++;
 721                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 722                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 723         }
 724
 725         if (misr & UART_IMR_RXSTALE) {
 726                 count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 727                         msm_port->old_snap_state;
 728                 msm_port->old_snap_state = 0;
 729         } else {
 730                 count = 4 * (msm_read(port, UART_RFWR));
 731                 msm_port->old_snap_state += count;
 732         }
 733
 734         /* TODO: Precise error reporting */
 735
 736         port->icount.rx += count;
 737
 738         while (count > 0) {
 739                 unsigned char buf[4];
 740                 int sysrq, r_count, i;
 741
 742                 sr = msm_read(port, UART_SR);
 743                 if ((sr & UART_SR_RX_READY) == 0) {
 744                         msm_port->old_snap_state -= count;
 745                         break;
 746                 }
 747
 748                 ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 749                 r_count = min_t(int, count, sizeof(buf));
 750
 751                 for (i = 0; i < r_count; i++) {
 752                         char flag = TTY_NORMAL;
 753
 754                         if (msm_port->break_detected && buf[i] == 0) {
 755                                 port->icount.brk++;
 756                                 flag = TTY_BREAK;
 757                                 msm_port->break_detected = false;
 758                                 if (uart_handle_break(port))
 759                                         continue;
 760                         }
 761
 762                         if (!(port->read_status_mask & UART_SR_RX_BREAK))
 763                                 flag = TTY_NORMAL;
 764
 765                         spin_unlock(&port->lock);
 766                         sysrq = uart_handle_sysrq_char(port, buf[i]);
 767                         spin_lock(&port->lock);
 768                         if (!sysrq)
 769                                 tty_insert_flip_char(tport, buf[i], flag);
 770                 }
 771                 count -= r_count;
 772         }
 773
 774         tty_flip_buffer_push(tport);
 775
 776         if (misr & (UART_IMR_RXSTALE))
 777                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 778         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 779         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 780
 781         /* Try to use DMA */
 782         msm_start_rx_dma(msm_port);
 783 }
 784
 785 static void msm_handle_rx(struct uart_port *port)
 786         __must_hold(&port->lock)
 787 {
 788         struct tty_port *tport = &port->state->port;
 789         unsigned int sr;
 790
 791         /*
 792          * Handle overrun. My understanding of the hardware is that overrun
 793          * is not tied to the RX buffer, so we handle the case out of band.
 794          */
 795         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 796                 port->icount.overrun++;
 797                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 798                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 799         }
 800
 801         /* and now the main RX loop */
 802         while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 803                 unsigned int c;
 804                 char flag = TTY_NORMAL;
 805                 int sysrq;
 806
 807                 c = msm_read(port, UART_RF);
 808
 809                 if (sr & UART_SR_RX_BREAK) {
 810                         port->icount.brk++;
 811                         if (uart_handle_break(port))
 812                                 continue;
 813                 } else if (sr & UART_SR_PAR_FRAME_ERR) {
 814                         port->icount.frame++;
 815                 } else {
 816                         port->icount.rx++;
 817                 }
 818
 819                 /* Mask conditions we're ignorning. */
 820                 sr &= port->read_status_mask;
 821
 822                 if (sr & UART_SR_RX_BREAK)
 823                         flag = TTY_BREAK;
 824                 else if (sr & UART_SR_PAR_FRAME_ERR)
 825                         flag = TTY_FRAME;
 826
 827                 spin_unlock(&port->lock);
 828                 sysrq = uart_handle_sysrq_char(port, c);
 829                 spin_lock(&port->lock);
 830                 if (!sysrq)
 831                         tty_insert_flip_char(tport, c, flag);
 832         }
 833
 834         tty_flip_buffer_push(tport);
 835 }
 836
 837 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 838 {
 839         struct circ_buf *xmit = &port->state->xmit;
 840         struct msm_port *msm_port = UART_TO_MSM(port);
 841         unsigned int num_chars;
 842         unsigned int tf_pointer = 0;
 843         void __iomem *tf;
 844
 845         if (msm_port->is_uartdm)
 846                 tf = port->membase + UARTDM_TF;
 847         else
 848                 tf = port->membase + UART_TF;
 849
 850         if (tx_count && msm_port->is_uartdm)
 851                 msm_reset_dm_count(port, tx_count);
 852
 853         while (tf_pointer < tx_count) {
 854                 int i;
 855                 char buf[4] = { 0 };
 856
 857                 if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 858                         break;
 859
 860                 if (msm_port->is_uartdm)
 861                         num_chars = min(tx_count - tf_pointer,
 862                                         (unsigned int)sizeof(buf));
 863                 else
 864                         num_chars = 1;
 865
 866                 for (i = 0; i < num_chars; i++) {
 867                         buf[i] = xmit->buf[xmit->tail + i];
 868                         port->icount.tx++;
 869                 }
 870
 871                 iowrite32_rep(tf, buf, 1);
 872                 xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 873                 tf_pointer += num_chars;
 874         }
 875
 876         /* disable tx interrupts if nothing more to send */
 877         if (uart_circ_empty(xmit))
 878                 msm_stop_tx(port);
 879
 880         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 881                 uart_write_wakeup(port);
 882 }
 883
 884 static void msm_handle_tx(struct uart_port *port)
 885 {
 886         struct msm_port *msm_port = UART_TO_MSM(port);
 887         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 888         struct msm_dma *dma = &msm_port->tx_dma;
 889         unsigned int pio_count, dma_count, dma_min;
 890         char buf[4] = { 0 };
 891         void __iomem *tf;
 892         int err = 0;
 893
 894         if (port->x_char) {
 895                 if (msm_port->is_uartdm)
 896                         tf = port->membase + UARTDM_TF;
 897                 else
 898                         tf = port->membase + UART_TF;
 899
 900                 buf[0] = port->x_char;
 901
 902                 if (msm_port->is_uartdm)
 903                         msm_reset_dm_count(port, 1);
 904
 905                 iowrite32_rep(tf, buf, 1);
 906                 port->icount.tx++;
 907                 port->x_char = 0;
 908                 return;
 909         }
 910
 911         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 912                 msm_stop_tx(port);
 913                 return;
 914         }
 915
 916         pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 917         dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 918
 919         dma_min = 1;    /* Always DMA */
 920         if (msm_port->is_uartdm > UARTDM_1P3) {
 921                 dma_count = UARTDM_TX_AIGN(dma_count);
 922                 dma_min = UARTDM_BURST_SIZE;
 923         } else {
 924                 if (dma_count > UARTDM_TX_MAX)
 925                         dma_count = UARTDM_TX_MAX;
 926         }
 927
 928         if (pio_count > port->fifosize)
 929                 pio_count = port->fifosize;
 930
 931         if (!dma->chan || dma_count < dma_min)
 932                 msm_handle_tx_pio(port, pio_count);
 933         else
 934                 err = msm_handle_tx_dma(msm_port, dma_count);
 935
 936         if (err)        /* fall back to PIO mode */
 937                 msm_handle_tx_pio(port, pio_count);
 938 }
 939
 940 static void msm_handle_delta_cts(struct uart_port *port)
 941 {
 942         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 943         port->icount.cts++;
 944         wake_up_interruptible(&port->state->port.delta_msr_wait);
 945 }
 946
 947 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 948 {
 949         struct uart_port *port = dev_id;
 950         struct msm_port *msm_port = UART_TO_MSM(port);
 951         struct msm_dma *dma = &msm_port->rx_dma;
 952         unsigned long flags;
 953         unsigned int misr;
 954         u32 val;
 955
 956         spin_lock_irqsave(&port->lock, flags);
 957         misr = msm_read(port, UART_MISR);
 958         msm_write(port, 0, UART_IMR); /* disable interrupt */
 959
 960         if (misr & UART_IMR_RXBREAK_START) {
 961                 msm_port->break_detected = true;
 962                 msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 963         }
 964
 965         if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 966                 if (dma->count) {
 967                         val = UART_CR_CMD_STALE_EVENT_DISABLE;
 968                         msm_write(port, val, UART_CR);
 969                         val = UART_CR_CMD_RESET_STALE_INT;
 970                         msm_write(port, val, UART_CR);
 971                         /*
 972                          * Flush DMA input fifo to memory, this will also
 973                          * trigger DMA RX completion
 974                          */
 975                         dmaengine_terminate_all(dma->chan);
 976                 } else if (msm_port->is_uartdm) {
 977                         msm_handle_rx_dm(port, misr);
 978                 } else {
 979                         msm_handle_rx(port);
 980                 }
 981         }
 982         if (misr & UART_IMR_TXLEV)
 983                 msm_handle_tx(port);
 984         if (misr & UART_IMR_DELTA_CTS)
 985                 msm_handle_delta_cts(port);
 986
 987         msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 988         spin_unlock_irqrestore(&port->lock, flags);
 989
 990         return IRQ_HANDLED;
 991 }
 992
 993 static unsigned int msm_tx_empty(struct uart_port *port)
 994 {
 995         return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 996 }
 997
 998 static unsigned int msm_get_mctrl(struct uart_port *port)
 999 {
1000         return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
1001 }
1002
1003 static void msm_reset(struct uart_port *port)
1004 {
1005         struct msm_port *msm_port = UART_TO_MSM(port);
1006         unsigned int mr;
1007
1008         /* reset everything */
1009         msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
1010         msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
1011         msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
1012         msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
1013         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
1014         msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1015         mr = msm_read(port, UART_MR1);
1016         mr &= ~UART_MR1_RX_RDY_CTL;
1017         msm_write(port, mr, UART_MR1);
1018
1019         /* Disable DM modes */
1020         if (msm_port->is_uartdm)
1021                 msm_write(port, 0, UARTDM_DMEN);
1022 }
1023
1024 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1025 {
1026         unsigned int mr;
1027
1028         mr = msm_read(port, UART_MR1);
1029
1030         if (!(mctrl & TIOCM_RTS)) {
1031                 mr &= ~UART_MR1_RX_RDY_CTL;
1032                 msm_write(port, mr, UART_MR1);
1033                 msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1034         } else {
1035                 mr |= UART_MR1_RX_RDY_CTL;
1036                 msm_write(port, mr, UART_MR1);
1037         }
1038 }
1039
1040 static void msm_break_ctl(struct uart_port *port, int break_ctl)
1041 {
1042         if (break_ctl)
1043                 msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1044         else
1045                 msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1046 }
1047
1048 struct msm_baud_map {
1049         u16     divisor;
1050         u8      code;
1051         u8      rxstale;
1052 };
1053
1054 static const struct msm_baud_map *
1055 msm_find_best_baud(struct uart_port *port, unsigned int baud,
1056                    unsigned long *rate)
1057 {
1058         struct msm_port *msm_port = UART_TO_MSM(port);
1059         unsigned int divisor, result;
1060         unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1061         const struct msm_baud_map *entry, *end, *best;
1062         static const struct msm_baud_map table[] = {
1063                 {    1, 0xff, 31 },
1064                 {    2, 0xee, 16 },
1065                 {    3, 0xdd,  8 },
1066                 {    4, 0xcc,  6 },
1067                 {    6, 0xbb,  6 },
1068                 {    8, 0xaa,  6 },
1069                 {   12, 0x99,  6 },
1070                 {   16, 0x88,  1 },
1071                 {   24, 0x77,  1 },
1072                 {   32, 0x66,  1 },
1073                 {   48, 0x55,  1 },
1074                 {   96, 0x44,  1 },
1075                 {  192, 0x33,  1 },
1076                 {  384, 0x22,  1 },
1077                 {  768, 0x11,  1 },
1078                 { 1536, 0x00,  1 },
1079         };
1080
1081         best = table; /* Default to smallest divider */
1082         target = clk_round_rate(msm_port->clk, 16 * baud);
1083         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1084
1085         end = table + ARRAY_SIZE(table);
1086         entry = table;
1087         while (entry < end) {
1088                 if (entry->divisor <= divisor) {
1089                         result = target / entry->divisor / 16;
1090                         diff = abs(result - baud);
1091
1092                         /* Keep track of best entry */
1093                         if (diff < best_diff) {
1094                                 best_diff = diff;
1095                                 best = entry;
1096                                 best_rate = target;
1097                         }
1098
1099                         if (result == baud)
1100                                 break;
1101                 } else if (entry->divisor > divisor) {
1102                         old = target;
1103                         target = clk_round_rate(msm_port->clk, old + 1);
1104                         /*
1105                          * The rate didn't get any faster so we can't do
1106                          * better at dividing it down
1107                          */
1108                         if (target == old)
1109                                 break;
1110
1111                         /* Start the divisor search over at this new rate */
1112                         entry = table;
1113                         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1114                         continue;
1115                 }
1116                 entry++;
1117         }
1118
1119         *rate = best_rate;
1120         return best;
1121 }
1122
1123 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1124                              unsigned long *saved_flags)
1125 {
1126         unsigned int rxstale, watermark, mask;
1127         struct msm_port *msm_port = UART_TO_MSM(port);
1128         const struct msm_baud_map *entry;
1129         unsigned long flags, rate;
1130
1131         flags = *saved_flags;
1132         spin_unlock_irqrestore(&port->lock, flags);
1133
1134         entry = msm_find_best_baud(port, baud, &rate);
1135         clk_set_rate(msm_port->clk, rate);
1136         baud = rate / 16 / entry->divisor;
1137
1138         spin_lock_irqsave(&port->lock, flags);
1139         *saved_flags = flags;
1140         port->uartclk = rate;
1141
1142         msm_write(port, entry->code, UART_CSR);
1143
1144         /* RX stale watermark */
1145         rxstale = entry->rxstale;
1146         watermark = UART_IPR_STALE_LSB & rxstale;
1147         if (msm_port->is_uartdm) {
1148                 mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1149         } else {
1150                 watermark |= UART_IPR_RXSTALE_LAST;
1151                 mask = UART_IPR_STALE_TIMEOUT_MSB;
1152         }
1153
1154         watermark |= mask & (rxstale << 2);
1155
1156         msm_write(port, watermark, UART_IPR);
1157
1158         /* set RX watermark */
1159         watermark = (port->fifosize * 3) / 4;
1160         msm_write(port, watermark, UART_RFWR);
1161
1162         /* set TX watermark */
1163         msm_write(port, 10, UART_TFWR);
1164
1165         msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1166         msm_reset(port);
1167
1168         /* Enable RX and TX */
1169         msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1170
1171         /* turn on RX and CTS interrupts */
1172         msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1173                         UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1174
1175         msm_write(port, msm_port->imr, UART_IMR);
1176
1177         if (msm_port->is_uartdm) {
1178                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1179                 msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1180                 msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1181         }
1182
1183         return baud;
1184 }
1185
1186 static void msm_init_clock(struct uart_port *port)
1187 {
1188         struct msm_port *msm_port = UART_TO_MSM(port);
1189
1190         clk_prepare_enable(msm_port->clk);
1191         clk_prepare_enable(msm_port->pclk);
1192         msm_serial_set_mnd_regs(port);
1193 }
1194
1195 static int msm_startup(struct uart_port *port)
1196 {
1197         struct msm_port *msm_port = UART_TO_MSM(port);
1198         unsigned int data, rfr_level, mask;
1199         int ret;
1200
1201         snprintf(msm_port->name, sizeof(msm_port->name),
1202                  "msm_serial%d", port->line);
1203
1204         msm_init_clock(port);
1205
1206         if (likely(port->fifosize > 12))
1207                 rfr_level = port->fifosize - 12;
1208         else
1209                 rfr_level = port->fifosize;
1210
1211         /* set automatic RFR level */
1212         data = msm_read(port, UART_MR1);
1213
1214         if (msm_port->is_uartdm)
1215                 mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1216         else
1217                 mask = UART_MR1_AUTO_RFR_LEVEL1;
1218
1219         data &= ~mask;
1220         data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1221         data |= mask & (rfr_level << 2);
1222         data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1223         msm_write(port, data, UART_MR1);
1224
1225         if (msm_port->is_uartdm) {
1226                 msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1227                 msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1228         }
1229
1230         ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1231                           msm_port->name, port);
1232         if (unlikely(ret))
1233                 goto err_irq;
1234
1235         return 0;
1236
1237 err_irq:
1238         if (msm_port->is_uartdm)
1239                 msm_release_dma(msm_port);
1240
1241         clk_disable_unprepare(msm_port->pclk);
1242         clk_disable_unprepare(msm_port->clk);
1243
1244         return ret;
1245 }
1246
1247 static void msm_shutdown(struct uart_port *port)
1248 {
1249         struct msm_port *msm_port = UART_TO_MSM(port);
1250
1251         msm_port->imr = 0;
1252         msm_write(port, 0, UART_IMR); /* disable interrupts */
1253
1254         if (msm_port->is_uartdm)
1255                 msm_release_dma(msm_port);
1256
1257         clk_disable_unprepare(msm_port->clk);
1258
1259         free_irq(port->irq, port);
1260 }
1261
1262 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1263                             struct ktermios *old)
1264 {
1265         struct msm_port *msm_port = UART_TO_MSM(port);
1266         struct msm_dma *dma = &msm_port->rx_dma;
1267         unsigned long flags;
1268         unsigned int baud, mr;
1269
1270         spin_lock_irqsave(&port->lock, flags);
1271
1272         if (dma->chan) /* Terminate if any */
1273                 msm_stop_dma(port, dma);
1274
1275         /* calculate and set baud rate */
1276         baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1277         baud = msm_set_baud_rate(port, baud, &flags);
1278         if (tty_termios_baud_rate(termios))
1279                 tty_termios_encode_baud_rate(termios, baud, baud);
1280
1281         /* calculate parity */
1282         mr = msm_read(port, UART_MR2);
1283         mr &= ~UART_MR2_PARITY_MODE;
1284         if (termios->c_cflag & PARENB) {
1285                 if (termios->c_cflag & PARODD)
1286                         mr |= UART_MR2_PARITY_MODE_ODD;
1287                 else if (termios->c_cflag & CMSPAR)
1288                         mr |= UART_MR2_PARITY_MODE_SPACE;
1289                 else
1290                         mr |= UART_MR2_PARITY_MODE_EVEN;
1291         }
1292
1293         /* calculate bits per char */
1294         mr &= ~UART_MR2_BITS_PER_CHAR;
1295         switch (termios->c_cflag & CSIZE) {
1296         case CS5:
1297                 mr |= UART_MR2_BITS_PER_CHAR_5;
1298                 break;
1299         case CS6:
1300                 mr |= UART_MR2_BITS_PER_CHAR_6;
1301                 break;
1302         case CS7:
1303                 mr |= UART_MR2_BITS_PER_CHAR_7;
1304                 break;
1305         case CS8:
1306         default:
1307                 mr |= UART_MR2_BITS_PER_CHAR_8;
1308                 break;
1309         }
1310
1311         /* calculate stop bits */
1312         mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1313         if (termios->c_cflag & CSTOPB)
1314                 mr |= UART_MR2_STOP_BIT_LEN_TWO;
1315         else
1316                 mr |= UART_MR2_STOP_BIT_LEN_ONE;
1317
1318         /* set parity, bits per char, and stop bit */
1319         msm_write(port, mr, UART_MR2);
1320
1321         /* calculate and set hardware flow control */
1322         mr = msm_read(port, UART_MR1);
1323         mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1324         if (termios->c_cflag & CRTSCTS) {
1325                 mr |= UART_MR1_CTS_CTL;
1326                 mr |= UART_MR1_RX_RDY_CTL;
1327         }
1328         msm_write(port, mr, UART_MR1);
1329
1330         /* Configure status bits to ignore based on termio flags. */
1331         port->read_status_mask = 0;
1332         if (termios->c_iflag & INPCK)
1333                 port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1334         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1335                 port->read_status_mask |= UART_SR_RX_BREAK;
1336
1337         uart_update_timeout(port, termios->c_cflag, baud);
1338
1339         /* Try to use DMA */
1340         msm_start_rx_dma(msm_port);
1341
1342         spin_unlock_irqrestore(&port->lock, flags);
1343 }
1344
1345 static const char *msm_type(struct uart_port *port)
1346 {
1347         return "MSM";
1348 }
1349
1350 static void msm_release_port(struct uart_port *port)
1351 {
1352         struct platform_device *pdev = to_platform_device(port->dev);
1353         struct resource *uart_resource;
1354         resource_size_t size;
1355
1356         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1357         if (unlikely(!uart_resource))
1358                 return;
1359         size = resource_size(uart_resource);
1360
1361         release_mem_region(port->mapbase, size);
1362         iounmap(port->membase);
1363         port->membase = NULL;
1364 }
1365
1366 static int msm_request_port(struct uart_port *port)
1367 {
1368         struct platform_device *pdev = to_platform_device(port->dev);
1369         struct resource *uart_resource;
1370         resource_size_t size;
1371         int ret;
1372
1373         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1374         if (unlikely(!uart_resource))
1375                 return -ENXIO;
1376
1377         size = resource_size(uart_resource);
1378
1379         if (!request_mem_region(port->mapbase, size, "msm_serial"))
1380                 return -EBUSY;
1381
1382         port->membase = ioremap(port->mapbase, size);
1383         if (!port->membase) {
1384                 ret = -EBUSY;
1385                 goto fail_release_port;
1386         }
1387
1388         return 0;
1389
1390 fail_release_port:
1391         release_mem_region(port->mapbase, size);
1392         return ret;
1393 }
1394
1395 static void msm_config_port(struct uart_port *port, int flags)
1396 {
1397         int ret;
1398
1399         if (flags & UART_CONFIG_TYPE) {
1400                 port->type = PORT_MSM;
1401                 ret = msm_request_port(port);
1402                 if (ret)
1403                         return;
1404         }
1405 }
1406
1407 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1408 {
1409         if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1410                 return -EINVAL;
1411         if (unlikely(port->irq != ser->irq))
1412                 return -EINVAL;
1413         return 0;
1414 }
1415
1416 static void msm_power(struct uart_port *port, unsigned int state,
1417                       unsigned int oldstate)
1418 {
1419         struct msm_port *msm_port = UART_TO_MSM(port);
1420
1421         switch (state) {
1422         case 0:
1423                 clk_prepare_enable(msm_port->clk);
1424                 clk_prepare_enable(msm_port->pclk);
1425                 break;
1426         case 3:
1427                 clk_disable_unprepare(msm_port->clk);
1428                 clk_disable_unprepare(msm_port->pclk);
1429                 break;
1430         default:
1431                 pr_err("msm_serial: Unknown PM state %d\n", state);
1432         }
1433 }
1434
1435 #ifdef CONFIG_CONSOLE_POLL
1436 static int msm_poll_get_char_single(struct uart_port *port)
1437 {
1438         struct msm_port *msm_port = UART_TO_MSM(port);
1439         unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1440
1441         if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1442                 return NO_POLL_CHAR;
1443
1444         return msm_read(port, rf_reg) & 0xff;
1445 }
1446
1447 static int msm_poll_get_char_dm(struct uart_port *port)
1448 {
1449         int c;
1450         static u32 slop;
1451         static int count;
1452         unsigned char *sp = (unsigned char *)&slop;
1453
1454         /* Check if a previous read had more than one char */
1455         if (count) {
1456                 c = sp[sizeof(slop) - count];
1457                 count--;
1458         /* Or if FIFO is empty */
1459         } else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1460                 /*
1461                  * If RX packing buffer has less than a word, force stale to
1462                  * push contents into RX FIFO
1463                  */
1464                 count = msm_read(port, UARTDM_RXFS);
1465                 count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1466                 if (count) {
1467                         msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1468                         slop = msm_read(port, UARTDM_RF);
1469                         c = sp[0];
1470                         count--;
1471                         msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1472                         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1473                         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1474                                   UART_CR);
1475                 } else {
1476                         c = NO_POLL_CHAR;
1477                 }
1478         /* FIFO has a word */
1479         } else {
1480                 slop = msm_read(port, UARTDM_RF);
1481                 c = sp[0];
1482                 count = sizeof(slop) - 1;
1483         }
1484
1485         return c;
1486 }
1487
1488 static int msm_poll_get_char(struct uart_port *port)
1489 {
1490         u32 imr;
1491         int c;
1492         struct msm_port *msm_port = UART_TO_MSM(port);
1493
1494         /* Disable all interrupts */
1495         imr = msm_read(port, UART_IMR);
1496         msm_write(port, 0, UART_IMR);
1497
1498         if (msm_port->is_uartdm)
1499                 c = msm_poll_get_char_dm(port);
1500         else
1501                 c = msm_poll_get_char_single(port);
1502
1503         /* Enable interrupts */
1504         msm_write(port, imr, UART_IMR);
1505
1506         return c;
1507 }
1508
1509 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1510 {
1511         u32 imr;
1512         struct msm_port *msm_port = UART_TO_MSM(port);
1513
1514         /* Disable all interrupts */
1515         imr = msm_read(port, UART_IMR);
1516         msm_write(port, 0, UART_IMR);
1517
1518         if (msm_port->is_uartdm)
1519                 msm_reset_dm_count(port, 1);
1520
1521         /* Wait until FIFO is empty */
1522         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1523                 cpu_relax();
1524
1525         /* Write a character */
1526         msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1527
1528         /* Wait until FIFO is empty */
1529         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1530                 cpu_relax();
1531
1532         /* Enable interrupts */
1533         msm_write(port, imr, UART_IMR);
1534 }
1535 #endif
1536
1537 static const struct uart_ops msm_uart_pops = {
1538         .tx_empty = msm_tx_empty,
1539         .set_mctrl = msm_set_mctrl,
1540         .get_mctrl = msm_get_mctrl,
1541         .stop_tx = msm_stop_tx,
1542         .start_tx = msm_start_tx,
1543         .stop_rx = msm_stop_rx,
1544         .enable_ms = msm_enable_ms,
1545         .break_ctl = msm_break_ctl,
1546         .startup = msm_startup,
1547         .shutdown = msm_shutdown,
1548         .set_termios = msm_set_termios,
1549         .type = msm_type,
1550         .release_port = msm_release_port,
1551         .request_port = msm_request_port,
1552         .config_port = msm_config_port,
1553         .verify_port = msm_verify_port,
1554         .pm = msm_power,
1555 #ifdef CONFIG_CONSOLE_POLL
1556         .poll_get_char  = msm_poll_get_char,
1557         .poll_put_char  = msm_poll_put_char,
1558 #endif
1559 };
1560
1561 static struct msm_port msm_uart_ports[] = {
1562         {
1563                 .uart = {
1564                         .iotype = UPIO_MEM,
1565                         .ops = &msm_uart_pops,
1566                         .flags = UPF_BOOT_AUTOCONF,
1567                         .fifosize = 64,
1568                         .line = 0,
1569                 },
1570         },
1571         {
1572                 .uart = {
1573                         .iotype = UPIO_MEM,
1574                         .ops = &msm_uart_pops,
1575                         .flags = UPF_BOOT_AUTOCONF,
1576                         .fifosize = 64,
1577                         .line = 1,
1578                 },
1579         },
1580         {
1581                 .uart = {
1582                         .iotype = UPIO_MEM,
1583                         .ops = &msm_uart_pops,
1584                         .flags = UPF_BOOT_AUTOCONF,
1585                         .fifosize = 64,
1586                         .line = 2,
1587                 },
1588         },
1589 };
1590
1591 #define UART_NR ARRAY_SIZE(msm_uart_ports)
1592
1593 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1594 {
1595         return &msm_uart_ports[line].uart;
1596 }
1597
1598 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1599 static void __msm_console_write(struct uart_port *port, const char *s,
1600                                 unsigned int count, bool is_uartdm)
1601 {
1602         unsigned long flags;
1603         int i;
1604         int num_newlines = 0;
1605         bool replaced = false;
1606         void __iomem *tf;
1607         int locked = 1;
1608
1609         if (is_uartdm)
1610                 tf = port->membase + UARTDM_TF;
1611         else
1612                 tf = port->membase + UART_TF;
1613
1614         /* Account for newlines that will get a carriage return added */
1615         for (i = 0; i < count; i++)
1616                 if (s[i] == '\n')
1617                         num_newlines++;
1618         count += num_newlines;
1619
1620         local_irq_save(flags);
1621
1622         if (port->sysrq)
1623                 locked = 0;
1624         else if (oops_in_progress)
1625                 locked = spin_trylock(&port->lock);
1626         else
1627                 spin_lock(&port->lock);
1628
1629         if (is_uartdm)
1630                 msm_reset_dm_count(port, count);
1631
1632         i = 0;
1633         while (i < count) {
1634                 int j;
1635                 unsigned int num_chars;
1636                 char buf[4] = { 0 };
1637
1638                 if (is_uartdm)
1639                         num_chars = min(count - i, (unsigned int)sizeof(buf));
1640                 else
1641                         num_chars = 1;
1642
1643                 for (j = 0; j < num_chars; j++) {
1644                         char c = *s;
1645
1646                         if (c == '\n' && !replaced) {
1647                                 buf[j] = '\r';
1648                                 j++;
1649                                 replaced = true;
1650                         }
1651                         if (j < num_chars) {
1652                                 buf[j] = c;
1653                                 s++;
1654                                 replaced = false;
1655                         }
1656                 }
1657
1658                 while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1659                         cpu_relax();
1660
1661                 iowrite32_rep(tf, buf, 1);
1662                 i += num_chars;
1663         }
1664
1665         if (locked)
1666                 spin_unlock(&port->lock);
1667
1668         local_irq_restore(flags);
1669 }
1670
1671 static void msm_console_write(struct console *co, const char *s,
1672                               unsigned int count)
1673 {
1674         struct uart_port *port;
1675         struct msm_port *msm_port;
1676
1677         BUG_ON(co->index < 0 || co->index >= UART_NR);
1678
1679         port = msm_get_port_from_line(co->index);
1680         msm_port = UART_TO_MSM(port);
1681
1682         __msm_console_write(port, s, count, msm_port->is_uartdm);
1683 }
1684
1685 static int msm_console_setup(struct console *co, char *options)
1686 {
1687         struct uart_port *port;
1688         int baud = 115200;
1689         int bits = 8;
1690         int parity = 'n';
1691         int flow = 'n';
1692
1693         if (unlikely(co->index >= UART_NR || co->index < 0))
1694                 return -ENXIO;
1695
1696         port = msm_get_port_from_line(co->index);
1697
1698         if (unlikely(!port->membase))
1699                 return -ENXIO;
1700
1701         msm_init_clock(port);
1702
1703         if (options)
1704                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1705
1706         pr_info("msm_serial: console setup on port #%d\n", port->line);
1707
1708         return uart_set_options(port, co, baud, parity, bits, flow);
1709 }
1710
1711 static void
1712 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1713 {
1714         struct earlycon_device *dev = con->data;
1715
1716         __msm_console_write(&dev->port, s, n, false);
1717 }
1718
1719 static int __init
1720 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1721 {
1722         if (!device->port.membase)
1723                 return -ENODEV;
1724
1725         device->con->write = msm_serial_early_write;
1726         return 0;
1727 }
1728 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1729                     msm_serial_early_console_setup);
1730
1731 static void
1732 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1733 {
1734         struct earlycon_device *dev = con->data;
1735
1736         __msm_console_write(&dev->port, s, n, true);
1737 }
1738
1739 static int __init
1740 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1741                                   const char *opt)
1742 {
1743         if (!device->port.membase)
1744                 return -ENODEV;
1745
1746         device->con->write = msm_serial_early_write_dm;
1747         return 0;
1748 }
1749 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1750                     msm_serial_early_console_setup_dm);
1751
1752 static struct uart_driver msm_uart_driver;
1753
1754 static struct console msm_console = {
1755         .name = "ttyMSM",
1756         .write = msm_console_write,
1757         .device = uart_console_device,
1758         .setup = msm_console_setup,
1759         .flags = CON_PRINTBUFFER,
1760         .index = -1,
1761         .data = &msm_uart_driver,
1762 };
1763
1764 #define MSM_CONSOLE     (&msm_console)
1765
1766 #else
1767 #define MSM_CONSOLE     NULL
1768 #endif
1769
1770 static struct uart_driver msm_uart_driver = {
1771         .owner = THIS_MODULE,
1772         .driver_name = "msm_serial",
1773         .dev_name = "ttyMSM",
1774         .nr = UART_NR,
1775         .cons = MSM_CONSOLE,
1776 };
1777
1778 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1779
1780 static const struct of_device_id msm_uartdm_table[] = {
1781         { .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1782         { .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1783         { .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1784         { .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1785         { }
1786 };
1787
1788 static int msm_serial_probe(struct platform_device *pdev)
1789 {
1790         struct msm_port *msm_port;
1791         struct resource *resource;
1792         struct uart_port *port;
1793         const struct of_device_id *id;
1794         int irq, line;
1795
1796         if (pdev->dev.of_node)
1797                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1798         else
1799                 line = pdev->id;
1800
1801         if (line < 0)
1802                 line = atomic_inc_return(&msm_uart_next_id) - 1;
1803
1804         if (unlikely(line < 0 || line >= UART_NR))
1805                 return -ENXIO;
1806
1807         dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1808
1809         port = msm_get_port_from_line(line);
1810         port->dev = &pdev->dev;
1811         msm_port = UART_TO_MSM(port);
1812
1813         id = of_match_device(msm_uartdm_table, &pdev->dev);
1814         if (id)
1815                 msm_port->is_uartdm = (unsigned long)id->data;
1816         else
1817                 msm_port->is_uartdm = 0;
1818
1819         msm_port->clk = devm_clk_get(&pdev->dev, "core");
1820         if (IS_ERR(msm_port->clk))
1821                 return PTR_ERR(msm_port->clk);
1822
1823         if (msm_port->is_uartdm) {
1824                 msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1825                 if (IS_ERR(msm_port->pclk))
1826                         return PTR_ERR(msm_port->pclk);
1827         }
1828
1829         port->uartclk = clk_get_rate(msm_port->clk);
1830         dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1831
1832         resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1833         if (unlikely(!resource))
1834                 return -ENXIO;
1835         port->mapbase = resource->start;
1836
1837         irq = platform_get_irq(pdev, 0);
1838         if (unlikely(irq < 0))
1839                 return -ENXIO;
1840         port->irq = irq;
1841         port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MSM_CONSOLE);
1842
1843         platform_set_drvdata(pdev, port);
1844
1845         return uart_add_one_port(&msm_uart_driver, port);
1846 }
1847
1848 static int msm_serial_remove(struct platform_device *pdev)
1849 {
1850         struct uart_port *port = platform_get_drvdata(pdev);
1851
1852         uart_remove_one_port(&msm_uart_driver, port);
1853
1854         return 0;
1855 }
1856
1857 static const struct of_device_id msm_match_table[] = {
1858         { .compatible = "qcom,msm-uart" },
1859         { .compatible = "qcom,msm-uartdm" },
1860         {}
1861 };
1862 MODULE_DEVICE_TABLE(of, msm_match_table);
1863
1864 static int __maybe_unused msm_serial_suspend(struct device *dev)
1865 {
1866         struct msm_port *port = dev_get_drvdata(dev);
1867
1868         uart_suspend_port(&msm_uart_driver, &port->uart);
1869
1870         return 0;
1871 }
1872
1873 static int __maybe_unused msm_serial_resume(struct device *dev)
1874 {
1875         struct msm_port *port = dev_get_drvdata(dev);
1876
1877         uart_resume_port(&msm_uart_driver, &port->uart);
1878
1879         return 0;
1880 }
1881
1882 static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1883         SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1884 };
1885
1886 static struct platform_driver msm_platform_driver = {
1887         .remove = msm_serial_remove,
1888         .probe = msm_serial_probe,
1889         .driver = {
1890                 .name = "msm_serial",
1891                 .pm = &msm_serial_dev_pm_ops,
1892                 .of_match_table = msm_match_table,
1893         },
1894 };
1895
1896 static int __init msm_serial_init(void)
1897 {
1898         int ret;
1899
1900         ret = uart_register_driver(&msm_uart_driver);
1901         if (unlikely(ret))
1902                 return ret;
1903
1904         ret = platform_driver_register(&msm_platform_driver);
1905         if (unlikely(ret))
1906                 uart_unregister_driver(&msm_uart_driver);
1907
1908         pr_info("msm_serial: driver initialized\n");
1909
1910         return ret;
1911 }
1912
1913 static void __exit msm_serial_exit(void)
1914 {
1915         platform_driver_unregister(&msm_platform_driver);
1916         uart_unregister_driver(&msm_uart_driver);
1917 }
1918
1919 module_init(msm_serial_init);
1920 module_exit(msm_serial_exit);
1921
1922 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1923 MODULE_DESCRIPTION("Driver for msm7x serial device");
1924 MODULE_LICENSE("GPL");