msm_serial.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for msm7k serial device and console
   4  *
   5  * Copyright (C) 2007 Google, Inc.
   6  * Author: Robert Love <rlove@google.com>
   7  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/atomic.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/module.h>
  15 #include <linux/io.h>
  16 #include <linux/ioport.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/init.h>
  19 #include <linux/console.h>
  20 #include <linux/tty.h>
  21 #include <linux/tty_flip.h>
  22 #include <linux/serial_core.h>
  23 #include <linux/slab.h>
  24 #include <linux/clk.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/delay.h>
  27 #include <linux/of.h>
  28 #include <linux/of_device.h>
  29 #include <linux/wait.h>
  30
  31 #define UART_MR1                        0x0000
  32
  33 #define UART_MR1_AUTO_RFR_LEVEL0        0x3F
  34 #define UART_MR1_AUTO_RFR_LEVEL1        0x3FF00
  35 #define UART_DM_MR1_AUTO_RFR_LEVEL1     0xFFFFFF00
  36 #define UART_MR1_RX_RDY_CTL             BIT(7)
  37 #define UART_MR1_CTS_CTL                BIT(6)
  38
  39 #define UART_MR2                        0x0004
  40 #define UART_MR2_ERROR_MODE             BIT(6)
  41 #define UART_MR2_BITS_PER_CHAR          0x30
  42 #define UART_MR2_BITS_PER_CHAR_5        (0x0 << 4)
  43 #define UART_MR2_BITS_PER_CHAR_6        (0x1 << 4)
  44 #define UART_MR2_BITS_PER_CHAR_7        (0x2 << 4)
  45 #define UART_MR2_BITS_PER_CHAR_8        (0x3 << 4)
  46 #define UART_MR2_STOP_BIT_LEN_ONE       (0x1 << 2)
  47 #define UART_MR2_STOP_BIT_LEN_TWO       (0x3 << 2)
  48 #define UART_MR2_PARITY_MODE_NONE       0x0
  49 #define UART_MR2_PARITY_MODE_ODD        0x1
  50 #define UART_MR2_PARITY_MODE_EVEN       0x2
  51 #define UART_MR2_PARITY_MODE_SPACE      0x3
  52 #define UART_MR2_PARITY_MODE            0x3
  53
  54 #define UART_CSR                        0x0008
  55
  56 #define UART_TF                         0x000C
  57 #define UARTDM_TF                       0x0070
  58
  59 #define UART_CR                         0x0010
  60 #define UART_CR_CMD_NULL                (0 << 4)
  61 #define UART_CR_CMD_RESET_RX            (1 << 4)
  62 #define UART_CR_CMD_RESET_TX            (2 << 4)
  63 #define UART_CR_CMD_RESET_ERR           (3 << 4)
  64 #define UART_CR_CMD_RESET_BREAK_INT     (4 << 4)
  65 #define UART_CR_CMD_START_BREAK         (5 << 4)
  66 #define UART_CR_CMD_STOP_BREAK          (6 << 4)
  67 #define UART_CR_CMD_RESET_CTS           (7 << 4)
  68 #define UART_CR_CMD_RESET_STALE_INT     (8 << 4)
  69 #define UART_CR_CMD_PACKET_MODE         (9 << 4)
  70 #define UART_CR_CMD_MODE_RESET          (12 << 4)
  71 #define UART_CR_CMD_SET_RFR             (13 << 4)
  72 #define UART_CR_CMD_RESET_RFR           (14 << 4)
  73 #define UART_CR_CMD_PROTECTION_EN       (16 << 4)
  74 #define UART_CR_CMD_STALE_EVENT_DISABLE (6 << 8)
  75 #define UART_CR_CMD_STALE_EVENT_ENABLE  (80 << 4)
  76 #define UART_CR_CMD_FORCE_STALE         (4 << 8)
  77 #define UART_CR_CMD_RESET_TX_READY      (3 << 8)
  78 #define UART_CR_TX_DISABLE              BIT(3)
  79 #define UART_CR_TX_ENABLE               BIT(2)
  80 #define UART_CR_RX_DISABLE              BIT(1)
  81 #define UART_CR_RX_ENABLE               BIT(0)
  82 #define UART_CR_CMD_RESET_RXBREAK_START ((1 << 11) | (2 << 4))
  83
  84 #define UART_IMR                        0x0014
  85 #define UART_IMR_TXLEV                  BIT(0)
  86 #define UART_IMR_RXSTALE                BIT(3)
  87 #define UART_IMR_RXLEV                  BIT(4)
  88 #define UART_IMR_DELTA_CTS              BIT(5)
  89 #define UART_IMR_CURRENT_CTS            BIT(6)
  90 #define UART_IMR_RXBREAK_START          BIT(10)
  91
  92 #define UART_IPR_RXSTALE_LAST           0x20
  93 #define UART_IPR_STALE_LSB              0x1F
  94 #define UART_IPR_STALE_TIMEOUT_MSB      0x3FF80
  95 #define UART_DM_IPR_STALE_TIMEOUT_MSB   0xFFFFFF80
  96
  97 #define UART_IPR                        0x0018
  98 #define UART_TFWR                       0x001C
  99 #define UART_RFWR                       0x0020
 100 #define UART_HCR                        0x0024
 101
 102 #define UART_MREG                       0x0028
 103 #define UART_NREG                       0x002C
 104 #define UART_DREG                       0x0030
 105 #define UART_MNDREG                     0x0034
 106 #define UART_IRDA                       0x0038
 107 #define UART_MISR_MODE                  0x0040
 108 #define UART_MISR_RESET                 0x0044
 109 #define UART_MISR_EXPORT                0x0048
 110 #define UART_MISR_VAL                   0x004C
 111 #define UART_TEST_CTRL                  0x0050
 112
 113 #define UART_SR                         0x0008
 114 #define UART_SR_HUNT_CHAR               BIT(7)
 115 #define UART_SR_RX_BREAK                BIT(6)
 116 #define UART_SR_PAR_FRAME_ERR           BIT(5)
 117 #define UART_SR_OVERRUN                 BIT(4)
 118 #define UART_SR_TX_EMPTY                BIT(3)
 119 #define UART_SR_TX_READY                BIT(2)
 120 #define UART_SR_RX_FULL                 BIT(1)
 121 #define UART_SR_RX_READY                BIT(0)
 122
 123 #define UART_RF                         0x000C
 124 #define UARTDM_RF                       0x0070
 125 #define UART_MISR                       0x0010
 126 #define UART_ISR                        0x0014
 127 #define UART_ISR_TX_READY               BIT(7)
 128
 129 #define UARTDM_RXFS                     0x50
 130 #define UARTDM_RXFS_BUF_SHIFT           0x7
 131 #define UARTDM_RXFS_BUF_MASK            0x7
 132
 133 #define UARTDM_DMEN                     0x3C
 134 #define UARTDM_DMEN_RX_SC_ENABLE        BIT(5)
 135 #define UARTDM_DMEN_TX_SC_ENABLE        BIT(4)
 136
 137 #define UARTDM_DMEN_TX_BAM_ENABLE       BIT(2)  /* UARTDM_1P4 */
 138 #define UARTDM_DMEN_TX_DM_ENABLE        BIT(0)  /* < UARTDM_1P4 */
 139
 140 #define UARTDM_DMEN_RX_BAM_ENABLE       BIT(3)  /* UARTDM_1P4 */
 141 #define UARTDM_DMEN_RX_DM_ENABLE        BIT(1)  /* < UARTDM_1P4 */
 142
 143 #define UARTDM_DMRX                     0x34
 144 #define UARTDM_NCF_TX                   0x40
 145 #define UARTDM_RX_TOTAL_SNAP            0x38
 146
 147 #define UARTDM_BURST_SIZE               16   /* in bytes */
 148 #define UARTDM_TX_AIGN(x)               ((x) & ~0x3) /* valid for > 1p3 */
 149 #define UARTDM_TX_MAX                   256   /* in bytes, valid for <= 1p3 */
 150 #define UARTDM_RX_SIZE                  (UART_XMIT_SIZE / 4)
 151
 152 enum {
 153         UARTDM_1P1 = 1,
 154         UARTDM_1P2,
 155         UARTDM_1P3,
 156         UARTDM_1P4,
 157 };
 158
 159 struct msm_dma {
 160         struct dma_chan         *chan;
 161         enum dma_data_direction dir;
 162         dma_addr_t              phys;
 163         unsigned char           *virt;
 164         dma_cookie_t            cookie;
 165         u32                     enable_bit;
 166         unsigned int            count;
 167         struct dma_async_tx_descriptor  *desc;
 168 };
 169
 170 struct msm_port {
 171         struct uart_port        uart;
 172         char                    name[16];
 173         struct clk              *clk;
 174         struct clk              *pclk;
 175         unsigned int            imr;
 176         int                     is_uartdm;
 177         unsigned int            old_snap_state;
 178         bool                    break_detected;
 179         struct msm_dma          tx_dma;
 180         struct msm_dma          rx_dma;
 181 };
 182
 183 #define UART_TO_MSM(uart_port)  container_of(uart_port, struct msm_port, uart)
 184
 185 static
 186 void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 187 {
 188         writel_relaxed(val, port->membase + off);
 189 }
 190
 191 static
 192 unsigned int msm_read(struct uart_port *port, unsigned int off)
 193 {
 194         return readl_relaxed(port->membase + off);
 195 }
 196
 197 /*
 198  * Setup the MND registers to use the TCXO clock.
 199  */
 200 static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 201 {
 202         msm_write(port, 0x06, UART_MREG);
 203         msm_write(port, 0xF1, UART_NREG);
 204         msm_write(port, 0x0F, UART_DREG);
 205         msm_write(port, 0x1A, UART_MNDREG);
 206         port->uartclk = 1843200;
 207 }
 208
 209 /*
 210  * Setup the MND registers to use the TCXO clock divided by 4.
 211  */
 212 static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 213 {
 214         msm_write(port, 0x18, UART_MREG);
 215         msm_write(port, 0xF6, UART_NREG);
 216         msm_write(port, 0x0F, UART_DREG);
 217         msm_write(port, 0x0A, UART_MNDREG);
 218         port->uartclk = 1843200;
 219 }
 220
 221 static void msm_serial_set_mnd_regs(struct uart_port *port)
 222 {
 223         struct msm_port *msm_port = UART_TO_MSM(port);
 224
 225         /*
 226          * These registers don't exist so we change the clk input rate
 227          * on uartdm hardware instead
 228          */
 229         if (msm_port->is_uartdm)
 230                 return;
 231
 232         if (port->uartclk == 19200000)
 233                 msm_serial_set_mnd_regs_tcxo(port);
 234         else if (port->uartclk == 4800000)
 235                 msm_serial_set_mnd_regs_tcxoby4(port);
 236 }
 237
 238 static void msm_handle_tx(struct uart_port *port);
 239 static void msm_start_rx_dma(struct msm_port *msm_port);
 240
 241 static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 242 {
 243         struct device *dev = port->dev;
 244         unsigned int mapped;
 245         u32 val;
 246
 247         mapped = dma->count;
 248         dma->count = 0;
 249
 250         dmaengine_terminate_all(dma->chan);
 251
 252         /*
 253          * DMA Stall happens if enqueue and flush command happens concurrently.
 254          * For example before changing the baud rate/protocol configuration and
 255          * sending flush command to ADM, disable the channel of UARTDM.
 256          * Note: should not reset the receiver here immediately as it is not
 257          * suggested to do disable/reset or reset/disable at the same time.
 258          */
 259         val = msm_read(port, UARTDM_DMEN);
 260         val &= ~dma->enable_bit;
 261         msm_write(port, val, UARTDM_DMEN);
 262
 263         if (mapped)
 264                 dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 265 }
 266
 267 static void msm_release_dma(struct msm_port *msm_port)
 268 {
 269         struct msm_dma *dma;
 270
 271         dma = &msm_port->tx_dma;
 272         if (dma->chan) {
 273                 msm_stop_dma(&msm_port->uart, dma);
 274                 dma_release_channel(dma->chan);
 275         }
 276
 277         memset(dma, 0, sizeof(*dma));
 278
 279         dma = &msm_port->rx_dma;
 280         if (dma->chan) {
 281                 msm_stop_dma(&msm_port->uart, dma);
 282                 dma_release_channel(dma->chan);
 283                 kfree(dma->virt);
 284         }
 285
 286         memset(dma, 0, sizeof(*dma));
 287 }
 288
 289 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 290 {
 291         struct device *dev = msm_port->uart.dev;
 292         struct dma_slave_config conf;
 293         struct msm_dma *dma;
 294         u32 crci = 0;
 295         int ret;
 296
 297         dma = &msm_port->tx_dma;
 298
 299         /* allocate DMA resources, if available */
 300         dma->chan = dma_request_chan(dev, "tx");
 301         if (IS_ERR(dma->chan))
 302                 goto no_tx;
 303
 304         of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 305
 306         memset(&conf, 0, sizeof(conf));
 307         conf.direction = DMA_MEM_TO_DEV;
 308         conf.device_fc = true;
 309         conf.dst_addr = base + UARTDM_TF;
 310         conf.dst_maxburst = UARTDM_BURST_SIZE;
 311         conf.slave_id = crci;
 312
 313         ret = dmaengine_slave_config(dma->chan, &conf);
 314         if (ret)
 315                 goto rel_tx;
 316
 317         dma->dir = DMA_TO_DEVICE;
 318
 319         if (msm_port->is_uartdm < UARTDM_1P4)
 320                 dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 321         else
 322                 dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 323
 324         return;
 325
 326 rel_tx:
 327         dma_release_channel(dma->chan);
 328 no_tx:
 329         memset(dma, 0, sizeof(*dma));
 330 }
 331
 332 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 333 {
 334         struct device *dev = msm_port->uart.dev;
 335         struct dma_slave_config conf;
 336         struct msm_dma *dma;
 337         u32 crci = 0;
 338         int ret;
 339
 340         dma = &msm_port->rx_dma;
 341
 342         /* allocate DMA resources, if available */
 343         dma->chan = dma_request_chan(dev, "rx");
 344         if (IS_ERR(dma->chan))
 345                 goto no_rx;
 346
 347         of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 348
 349         dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 350         if (!dma->virt)
 351                 goto rel_rx;
 352
 353         memset(&conf, 0, sizeof(conf));
 354         conf.direction = DMA_DEV_TO_MEM;
 355         conf.device_fc = true;
 356         conf.src_addr = base + UARTDM_RF;
 357         conf.src_maxburst = UARTDM_BURST_SIZE;
 358         conf.slave_id = crci;
 359
 360         ret = dmaengine_slave_config(dma->chan, &conf);
 361         if (ret)
 362                 goto err;
 363
 364         dma->dir = DMA_FROM_DEVICE;
 365
 366         if (msm_port->is_uartdm < UARTDM_1P4)
 367                 dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 368         else
 369                 dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 370
 371         return;
 372 err:
 373         kfree(dma->virt);
 374 rel_rx:
 375         dma_release_channel(dma->chan);
 376 no_rx:
 377         memset(dma, 0, sizeof(*dma));
 378 }
 379
 380 static inline void msm_wait_for_xmitr(struct uart_port *port)
 381 {
 382         unsigned int timeout = 500000;
 383
 384         while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 385                 if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 386                         break;
 387                 udelay(1);
 388                 if (!timeout--)
 389                         break;
 390         }
 391         msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 392 }
 393
 394 static void msm_stop_tx(struct uart_port *port)
 395 {
 396         struct msm_port *msm_port = UART_TO_MSM(port);
 397
 398         msm_port->imr &= ~UART_IMR_TXLEV;
 399         msm_write(port, msm_port->imr, UART_IMR);
 400 }
 401
 402 static void msm_start_tx(struct uart_port *port)
 403 {
 404         struct msm_port *msm_port = UART_TO_MSM(port);
 405         struct msm_dma *dma = &msm_port->tx_dma;
 406
 407         /* Already started in DMA mode */
 408         if (dma->count)
 409                 return;
 410
 411         msm_port->imr |= UART_IMR_TXLEV;
 412         msm_write(port, msm_port->imr, UART_IMR);
 413 }
 414
 415 static void msm_reset_dm_count(struct uart_port *port, int count)
 416 {
 417         msm_wait_for_xmitr(port);
 418         msm_write(port, count, UARTDM_NCF_TX);
 419         msm_read(port, UARTDM_NCF_TX);
 420 }
 421
 422 static void msm_complete_tx_dma(void *args)
 423 {
 424         struct msm_port *msm_port = args;
 425         struct uart_port *port = &msm_port->uart;
 426         struct circ_buf *xmit = &port->state->xmit;
 427         struct msm_dma *dma = &msm_port->tx_dma;
 428         struct dma_tx_state state;
 429         enum dma_status status;
 430         unsigned long flags;
 431         unsigned int count;
 432         u32 val;
 433
 434         spin_lock_irqsave(&port->lock, flags);
 435
 436         /* Already stopped */
 437         if (!dma->count)
 438                 goto done;
 439
 440         status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
 441
 442         dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 443
 444         val = msm_read(port, UARTDM_DMEN);
 445         val &= ~dma->enable_bit;
 446         msm_write(port, val, UARTDM_DMEN);
 447
 448         if (msm_port->is_uartdm > UARTDM_1P3) {
 449                 msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 450                 msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 451         }
 452
 453         count = dma->count - state.residue;
 454         port->icount.tx += count;
 455         dma->count = 0;
 456
 457         xmit->tail += count;
 458         xmit->tail &= UART_XMIT_SIZE - 1;
 459
 460         /* Restore "Tx FIFO below watermark" interrupt */
 461         msm_port->imr |= UART_IMR_TXLEV;
 462         msm_write(port, msm_port->imr, UART_IMR);
 463
 464         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 465                 uart_write_wakeup(port);
 466
 467         msm_handle_tx(port);
 468 done:
 469         spin_unlock_irqrestore(&port->lock, flags);
 470 }
 471
 472 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 473 {
 474         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 475         struct uart_port *port = &msm_port->uart;
 476         struct msm_dma *dma = &msm_port->tx_dma;
 477         void *cpu_addr;
 478         int ret;
 479         u32 val;
 480
 481         cpu_addr = &xmit->buf[xmit->tail];
 482
 483         dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 484         ret = dma_mapping_error(port->dev, dma->phys);
 485         if (ret)
 486                 return ret;
 487
 488         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 489                                                 count, DMA_MEM_TO_DEV,
 490                                                 DMA_PREP_INTERRUPT |
 491                                                 DMA_PREP_FENCE);
 492         if (!dma->desc) {
 493                 ret = -EIO;
 494                 goto unmap;
 495         }
 496
 497         dma->desc->callback = msm_complete_tx_dma;
 498         dma->desc->callback_param = msm_port;
 499
 500         dma->cookie = dmaengine_submit(dma->desc);
 501         ret = dma_submit_error(dma->cookie);
 502         if (ret)
 503                 goto unmap;
 504
 505         /*
 506          * Using DMA complete for Tx FIFO reload, no need for
 507          * "Tx FIFO below watermark" one, disable it
 508          */
 509         msm_port->imr &= ~UART_IMR_TXLEV;
 510         msm_write(port, msm_port->imr, UART_IMR);
 511
 512         dma->count = count;
 513
 514         val = msm_read(port, UARTDM_DMEN);
 515         val |= dma->enable_bit;
 516
 517         if (msm_port->is_uartdm < UARTDM_1P4)
 518                 msm_write(port, val, UARTDM_DMEN);
 519
 520         msm_reset_dm_count(port, count);
 521
 522         if (msm_port->is_uartdm > UARTDM_1P3)
 523                 msm_write(port, val, UARTDM_DMEN);
 524
 525         dma_async_issue_pending(dma->chan);
 526         return 0;
 527 unmap:
 528         dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 529         return ret;
 530 }
 531
 532 static void msm_complete_rx_dma(void *args)
 533 {
 534         struct msm_port *msm_port = args;
 535         struct uart_port *port = &msm_port->uart;
 536         struct tty_port *tport = &port->state->port;
 537         struct msm_dma *dma = &msm_port->rx_dma;
 538         int count = 0, i, sysrq;
 539         unsigned long flags;
 540         u32 val;
 541
 542         spin_lock_irqsave(&port->lock, flags);
 543
 544         /* Already stopped */
 545         if (!dma->count)
 546                 goto done;
 547
 548         val = msm_read(port, UARTDM_DMEN);
 549         val &= ~dma->enable_bit;
 550         msm_write(port, val, UARTDM_DMEN);
 551
 552         if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 553                 port->icount.overrun++;
 554                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 555                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 556         }
 557
 558         count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 559
 560         port->icount.rx += count;
 561
 562         dma->count = 0;
 563
 564         dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 565
 566         for (i = 0; i < count; i++) {
 567                 char flag = TTY_NORMAL;
 568
 569                 if (msm_port->break_detected && dma->virt[i] == 0) {
 570                         port->icount.brk++;
 571                         flag = TTY_BREAK;
 572                         msm_port->break_detected = false;
 573                         if (uart_handle_break(port))
 574                                 continue;
 575                 }
 576
 577                 if (!(port->read_status_mask & UART_SR_RX_BREAK))
 578                         flag = TTY_NORMAL;
 579
 580                 spin_unlock_irqrestore(&port->lock, flags);
 581                 sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 582                 spin_lock_irqsave(&port->lock, flags);
 583                 if (!sysrq)
 584                         tty_insert_flip_char(tport, dma->virt[i], flag);
 585         }
 586
 587         msm_start_rx_dma(msm_port);
 588 done:
 589         spin_unlock_irqrestore(&port->lock, flags);
 590
 591         if (count)
 592                 tty_flip_buffer_push(tport);
 593 }
 594
 595 static void msm_start_rx_dma(struct msm_port *msm_port)
 596 {
 597         struct msm_dma *dma = &msm_port->rx_dma;
 598         struct uart_port *uart = &msm_port->uart;
 599         u32 val;
 600         int ret;
 601
 602         if (IS_ENABLED(CONFIG_CONSOLE_POLL))
 603                 return;
 604
 605         if (!dma->chan)
 606                 return;
 607
 608         dma->phys = dma_map_single(uart->dev, dma->virt,
 609                                    UARTDM_RX_SIZE, dma->dir);
 610         ret = dma_mapping_error(uart->dev, dma->phys);
 611         if (ret)
 612                 goto sw_mode;
 613
 614         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 615                                                 UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 616                                                 DMA_PREP_INTERRUPT);
 617         if (!dma->desc)
 618                 goto unmap;
 619
 620         dma->desc->callback = msm_complete_rx_dma;
 621         dma->desc->callback_param = msm_port;
 622
 623         dma->cookie = dmaengine_submit(dma->desc);
 624         ret = dma_submit_error(dma->cookie);
 625         if (ret)
 626                 goto unmap;
 627         /*
 628          * Using DMA for FIFO off-load, no need for "Rx FIFO over
 629          * watermark" or "stale" interrupts, disable them
 630          */
 631         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 632
 633         /*
 634          * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 635          * we need RXSTALE to flush input DMA fifo to memory
 636          */
 637         if (msm_port->is_uartdm < UARTDM_1P4)
 638                 msm_port->imr |= UART_IMR_RXSTALE;
 639
 640         msm_write(uart, msm_port->imr, UART_IMR);
 641
 642         dma->count = UARTDM_RX_SIZE;
 643
 644         dma_async_issue_pending(dma->chan);
 645
 646         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 647         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 648
 649         val = msm_read(uart, UARTDM_DMEN);
 650         val |= dma->enable_bit;
 651
 652         if (msm_port->is_uartdm < UARTDM_1P4)
 653                 msm_write(uart, val, UARTDM_DMEN);
 654
 655         msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 656
 657         if (msm_port->is_uartdm > UARTDM_1P3)
 658                 msm_write(uart, val, UARTDM_DMEN);
 659
 660         return;
 661 unmap:
 662         dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 663
 664 sw_mode:
 665         /*
 666          * Switch from DMA to SW/FIFO mode. After clearing Rx BAM (UARTDM_DMEN),
 667          * receiver must be reset.
 668          */
 669         msm_write(uart, UART_CR_CMD_RESET_RX, UART_CR);
 670         msm_write(uart, UART_CR_RX_ENABLE, UART_CR);
 671
 672         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 673         msm_write(uart, 0xFFFFFF, UARTDM_DMRX);
 674         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 675
 676         /* Re-enable RX interrupts */
 677         msm_port->imr |= (UART_IMR_RXLEV | UART_IMR_RXSTALE);
 678         msm_write(uart, msm_port->imr, UART_IMR);
 679 }
 680
 681 static void msm_stop_rx(struct uart_port *port)
 682 {
 683         struct msm_port *msm_port = UART_TO_MSM(port);
 684         struct msm_dma *dma = &msm_port->rx_dma;
 685
 686         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 687         msm_write(port, msm_port->imr, UART_IMR);
 688
 689         if (dma->chan)
 690                 msm_stop_dma(port, dma);
 691 }
 692
 693 static void msm_enable_ms(struct uart_port *port)
 694 {
 695         struct msm_port *msm_port = UART_TO_MSM(port);
 696
 697         msm_port->imr |= UART_IMR_DELTA_CTS;
 698         msm_write(port, msm_port->imr, UART_IMR);
 699 }
 700
 701 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 702         __must_hold(&port->lock)
 703 {
 704         struct tty_port *tport = &port->state->port;
 705         unsigned int sr;
 706         int count = 0;
 707         struct msm_port *msm_port = UART_TO_MSM(port);
 708
 709         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 710                 port->icount.overrun++;
 711                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 712                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 713         }
 714
 715         if (misr & UART_IMR_RXSTALE) {
 716                 count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 717                         msm_port->old_snap_state;
 718                 msm_port->old_snap_state = 0;
 719         } else {
 720                 count = 4 * (msm_read(port, UART_RFWR));
 721                 msm_port->old_snap_state += count;
 722         }
 723
 724         /* TODO: Precise error reporting */
 725
 726         port->icount.rx += count;
 727
 728         while (count > 0) {
 729                 unsigned char buf[4];
 730                 int sysrq, r_count, i;
 731
 732                 sr = msm_read(port, UART_SR);
 733                 if ((sr & UART_SR_RX_READY) == 0) {
 734                         msm_port->old_snap_state -= count;
 735                         break;
 736                 }
 737
 738                 ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 739                 r_count = min_t(int, count, sizeof(buf));
 740
 741                 for (i = 0; i < r_count; i++) {
 742                         char flag = TTY_NORMAL;
 743
 744                         if (msm_port->break_detected && buf[i] == 0) {
 745                                 port->icount.brk++;
 746                                 flag = TTY_BREAK;
 747                                 msm_port->break_detected = false;
 748                                 if (uart_handle_break(port))
 749                                         continue;
 750                         }
 751
 752                         if (!(port->read_status_mask & UART_SR_RX_BREAK))
 753                                 flag = TTY_NORMAL;
 754
 755                         spin_unlock(&port->lock);
 756                         sysrq = uart_handle_sysrq_char(port, buf[i]);
 757                         spin_lock(&port->lock);
 758                         if (!sysrq)
 759                                 tty_insert_flip_char(tport, buf[i], flag);
 760                 }
 761                 count -= r_count;
 762         }
 763
 764         spin_unlock(&port->lock);
 765         tty_flip_buffer_push(tport);
 766         spin_lock(&port->lock);
 767
 768         if (misr & (UART_IMR_RXSTALE))
 769                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 770         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 771         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 772
 773         /* Try to use DMA */
 774         msm_start_rx_dma(msm_port);
 775 }
 776
 777 static void msm_handle_rx(struct uart_port *port)
 778         __must_hold(&port->lock)
 779 {
 780         struct tty_port *tport = &port->state->port;
 781         unsigned int sr;
 782
 783         /*
 784          * Handle overrun. My understanding of the hardware is that overrun
 785          * is not tied to the RX buffer, so we handle the case out of band.
 786          */
 787         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 788                 port->icount.overrun++;
 789                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 790                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 791         }
 792
 793         /* and now the main RX loop */
 794         while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 795                 unsigned int c;
 796                 char flag = TTY_NORMAL;
 797                 int sysrq;
 798
 799                 c = msm_read(port, UART_RF);
 800
 801                 if (sr & UART_SR_RX_BREAK) {
 802                         port->icount.brk++;
 803                         if (uart_handle_break(port))
 804                                 continue;
 805                 } else if (sr & UART_SR_PAR_FRAME_ERR) {
 806                         port->icount.frame++;
 807                 } else {
 808                         port->icount.rx++;
 809                 }
 810
 811                 /* Mask conditions we're ignorning. */
 812                 sr &= port->read_status_mask;
 813
 814                 if (sr & UART_SR_RX_BREAK)
 815                         flag = TTY_BREAK;
 816                 else if (sr & UART_SR_PAR_FRAME_ERR)
 817                         flag = TTY_FRAME;
 818
 819                 spin_unlock(&port->lock);
 820                 sysrq = uart_handle_sysrq_char(port, c);
 821                 spin_lock(&port->lock);
 822                 if (!sysrq)
 823                         tty_insert_flip_char(tport, c, flag);
 824         }
 825
 826         spin_unlock(&port->lock);
 827         tty_flip_buffer_push(tport);
 828         spin_lock(&port->lock);
 829 }
 830
 831 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 832 {
 833         struct circ_buf *xmit = &port->state->xmit;
 834         struct msm_port *msm_port = UART_TO_MSM(port);
 835         unsigned int num_chars;
 836         unsigned int tf_pointer = 0;
 837         void __iomem *tf;
 838
 839         if (msm_port->is_uartdm)
 840                 tf = port->membase + UARTDM_TF;
 841         else
 842                 tf = port->membase + UART_TF;
 843
 844         if (tx_count && msm_port->is_uartdm)
 845                 msm_reset_dm_count(port, tx_count);
 846
 847         while (tf_pointer < tx_count) {
 848                 int i;
 849                 char buf[4] = { 0 };
 850
 851                 if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 852                         break;
 853
 854                 if (msm_port->is_uartdm)
 855                         num_chars = min(tx_count - tf_pointer,
 856                                         (unsigned int)sizeof(buf));
 857                 else
 858                         num_chars = 1;
 859
 860                 for (i = 0; i < num_chars; i++) {
 861                         buf[i] = xmit->buf[xmit->tail + i];
 862                         port->icount.tx++;
 863                 }
 864
 865                 iowrite32_rep(tf, buf, 1);
 866                 xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 867                 tf_pointer += num_chars;
 868         }
 869
 870         /* disable tx interrupts if nothing more to send */
 871         if (uart_circ_empty(xmit))
 872                 msm_stop_tx(port);
 873
 874         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 875                 uart_write_wakeup(port);
 876 }
 877
 878 static void msm_handle_tx(struct uart_port *port)
 879 {
 880         struct msm_port *msm_port = UART_TO_MSM(port);
 881         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 882         struct msm_dma *dma = &msm_port->tx_dma;
 883         unsigned int pio_count, dma_count, dma_min;
 884         char buf[4] = { 0 };
 885         void __iomem *tf;
 886         int err = 0;
 887
 888         if (port->x_char) {
 889                 if (msm_port->is_uartdm)
 890                         tf = port->membase + UARTDM_TF;
 891                 else
 892                         tf = port->membase + UART_TF;
 893
 894                 buf[0] = port->x_char;
 895
 896                 if (msm_port->is_uartdm)
 897                         msm_reset_dm_count(port, 1);
 898
 899                 iowrite32_rep(tf, buf, 1);
 900                 port->icount.tx++;
 901                 port->x_char = 0;
 902                 return;
 903         }
 904
 905         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 906                 msm_stop_tx(port);
 907                 return;
 908         }
 909
 910         pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 911         dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 912
 913         dma_min = 1;    /* Always DMA */
 914         if (msm_port->is_uartdm > UARTDM_1P3) {
 915                 dma_count = UARTDM_TX_AIGN(dma_count);
 916                 dma_min = UARTDM_BURST_SIZE;
 917         } else {
 918                 if (dma_count > UARTDM_TX_MAX)
 919                         dma_count = UARTDM_TX_MAX;
 920         }
 921
 922         if (pio_count > port->fifosize)
 923                 pio_count = port->fifosize;
 924
 925         if (!dma->chan || dma_count < dma_min)
 926                 msm_handle_tx_pio(port, pio_count);
 927         else
 928                 err = msm_handle_tx_dma(msm_port, dma_count);
 929
 930         if (err)        /* fall back to PIO mode */
 931                 msm_handle_tx_pio(port, pio_count);
 932 }
 933
 934 static void msm_handle_delta_cts(struct uart_port *port)
 935 {
 936         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 937         port->icount.cts++;
 938         wake_up_interruptible(&port->state->port.delta_msr_wait);
 939 }
 940
 941 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 942 {
 943         struct uart_port *port = dev_id;
 944         struct msm_port *msm_port = UART_TO_MSM(port);
 945         struct msm_dma *dma = &msm_port->rx_dma;
 946         unsigned long flags;
 947         unsigned int misr;
 948         u32 val;
 949
 950         spin_lock_irqsave(&port->lock, flags);
 951         misr = msm_read(port, UART_MISR);
 952         msm_write(port, 0, UART_IMR); /* disable interrupt */
 953
 954         if (misr & UART_IMR_RXBREAK_START) {
 955                 msm_port->break_detected = true;
 956                 msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 957         }
 958
 959         if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 960                 if (dma->count) {
 961                         val = UART_CR_CMD_STALE_EVENT_DISABLE;
 962                         msm_write(port, val, UART_CR);
 963                         val = UART_CR_CMD_RESET_STALE_INT;
 964                         msm_write(port, val, UART_CR);
 965                         /*
 966                          * Flush DMA input fifo to memory, this will also
 967                          * trigger DMA RX completion
 968                          */
 969                         dmaengine_terminate_all(dma->chan);
 970                 } else if (msm_port->is_uartdm) {
 971                         msm_handle_rx_dm(port, misr);
 972                 } else {
 973                         msm_handle_rx(port);
 974                 }
 975         }
 976         if (misr & UART_IMR_TXLEV)
 977                 msm_handle_tx(port);
 978         if (misr & UART_IMR_DELTA_CTS)
 979                 msm_handle_delta_cts(port);
 980
 981         msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 982         spin_unlock_irqrestore(&port->lock, flags);
 983
 984         return IRQ_HANDLED;
 985 }
 986
 987 static unsigned int msm_tx_empty(struct uart_port *port)
 988 {
 989         return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 990 }
 991
 992 static unsigned int msm_get_mctrl(struct uart_port *port)
 993 {
 994         return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
 995 }
 996
 997 static void msm_reset(struct uart_port *port)
 998 {
 999         struct msm_port *msm_port = UART_TO_MSM(port);
1000         unsigned int mr;
1001
1002         /* reset everything */
1003         msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
1004         msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
1005         msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
1006         msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
1007         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
1008         msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1009         mr = msm_read(port, UART_MR1);
1010         mr &= ~UART_MR1_RX_RDY_CTL;
1011         msm_write(port, mr, UART_MR1);
1012
1013         /* Disable DM modes */
1014         if (msm_port->is_uartdm)
1015                 msm_write(port, 0, UARTDM_DMEN);
1016 }
1017
1018 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1019 {
1020         unsigned int mr;
1021
1022         mr = msm_read(port, UART_MR1);
1023
1024         if (!(mctrl & TIOCM_RTS)) {
1025                 mr &= ~UART_MR1_RX_RDY_CTL;
1026                 msm_write(port, mr, UART_MR1);
1027                 msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1028         } else {
1029                 mr |= UART_MR1_RX_RDY_CTL;
1030                 msm_write(port, mr, UART_MR1);
1031         }
1032 }
1033
1034 static void msm_break_ctl(struct uart_port *port, int break_ctl)
1035 {
1036         if (break_ctl)
1037                 msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1038         else
1039                 msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1040 }
1041
1042 struct msm_baud_map {
1043         u16     divisor;
1044         u8      code;
1045         u8      rxstale;
1046 };
1047
1048 static const struct msm_baud_map *
1049 msm_find_best_baud(struct uart_port *port, unsigned int baud,
1050                    unsigned long *rate)
1051 {
1052         struct msm_port *msm_port = UART_TO_MSM(port);
1053         unsigned int divisor, result;
1054         unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1055         const struct msm_baud_map *entry, *end, *best;
1056         static const struct msm_baud_map table[] = {
1057                 {    1, 0xff, 31 },
1058                 {    2, 0xee, 16 },
1059                 {    3, 0xdd,  8 },
1060                 {    4, 0xcc,  6 },
1061                 {    6, 0xbb,  6 },
1062                 {    8, 0xaa,  6 },
1063                 {   12, 0x99,  6 },
1064                 {   16, 0x88,  1 },
1065                 {   24, 0x77,  1 },
1066                 {   32, 0x66,  1 },
1067                 {   48, 0x55,  1 },
1068                 {   96, 0x44,  1 },
1069                 {  192, 0x33,  1 },
1070                 {  384, 0x22,  1 },
1071                 {  768, 0x11,  1 },
1072                 { 1536, 0x00,  1 },
1073         };
1074
1075         best = table; /* Default to smallest divider */
1076         target = clk_round_rate(msm_port->clk, 16 * baud);
1077         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1078
1079         end = table + ARRAY_SIZE(table);
1080         entry = table;
1081         while (entry < end) {
1082                 if (entry->divisor <= divisor) {
1083                         result = target / entry->divisor / 16;
1084                         diff = abs(result - baud);
1085
1086                         /* Keep track of best entry */
1087                         if (diff < best_diff) {
1088                                 best_diff = diff;
1089                                 best = entry;
1090                                 best_rate = target;
1091                         }
1092
1093                         if (result == baud)
1094                                 break;
1095                 } else if (entry->divisor > divisor) {
1096                         old = target;
1097                         target = clk_round_rate(msm_port->clk, old + 1);
1098                         /*
1099                          * The rate didn't get any faster so we can't do
1100                          * better at dividing it down
1101                          */
1102                         if (target == old)
1103                                 break;
1104
1105                         /* Start the divisor search over at this new rate */
1106                         entry = table;
1107                         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1108                         continue;
1109                 }
1110                 entry++;
1111         }
1112
1113         *rate = best_rate;
1114         return best;
1115 }
1116
1117 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1118                              unsigned long *saved_flags)
1119 {
1120         unsigned int rxstale, watermark, mask;
1121         struct msm_port *msm_port = UART_TO_MSM(port);
1122         const struct msm_baud_map *entry;
1123         unsigned long flags, rate;
1124
1125         flags = *saved_flags;
1126         spin_unlock_irqrestore(&port->lock, flags);
1127
1128         entry = msm_find_best_baud(port, baud, &rate);
1129         clk_set_rate(msm_port->clk, rate);
1130         baud = rate / 16 / entry->divisor;
1131
1132         spin_lock_irqsave(&port->lock, flags);
1133         *saved_flags = flags;
1134         port->uartclk = rate;
1135
1136         msm_write(port, entry->code, UART_CSR);
1137
1138         /* RX stale watermark */
1139         rxstale = entry->rxstale;
1140         watermark = UART_IPR_STALE_LSB & rxstale;
1141         if (msm_port->is_uartdm) {
1142                 mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1143         } else {
1144                 watermark |= UART_IPR_RXSTALE_LAST;
1145                 mask = UART_IPR_STALE_TIMEOUT_MSB;
1146         }
1147
1148         watermark |= mask & (rxstale << 2);
1149
1150         msm_write(port, watermark, UART_IPR);
1151
1152         /* set RX watermark */
1153         watermark = (port->fifosize * 3) / 4;
1154         msm_write(port, watermark, UART_RFWR);
1155
1156         /* set TX watermark */
1157         msm_write(port, 10, UART_TFWR);
1158
1159         msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1160         msm_reset(port);
1161
1162         /* Enable RX and TX */
1163         msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1164
1165         /* turn on RX and CTS interrupts */
1166         msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1167                         UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1168
1169         msm_write(port, msm_port->imr, UART_IMR);
1170
1171         if (msm_port->is_uartdm) {
1172                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1173                 msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1174                 msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1175         }
1176
1177         return baud;
1178 }
1179
1180 static void msm_init_clock(struct uart_port *port)
1181 {
1182         struct msm_port *msm_port = UART_TO_MSM(port);
1183
1184         clk_prepare_enable(msm_port->clk);
1185         clk_prepare_enable(msm_port->pclk);
1186         msm_serial_set_mnd_regs(port);
1187 }
1188
1189 static int msm_startup(struct uart_port *port)
1190 {
1191         struct msm_port *msm_port = UART_TO_MSM(port);
1192         unsigned int data, rfr_level, mask;
1193         int ret;
1194
1195         snprintf(msm_port->name, sizeof(msm_port->name),
1196                  "msm_serial%d", port->line);
1197
1198         msm_init_clock(port);
1199
1200         if (likely(port->fifosize > 12))
1201                 rfr_level = port->fifosize - 12;
1202         else
1203                 rfr_level = port->fifosize;
1204
1205         /* set automatic RFR level */
1206         data = msm_read(port, UART_MR1);
1207
1208         if (msm_port->is_uartdm)
1209                 mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1210         else
1211                 mask = UART_MR1_AUTO_RFR_LEVEL1;
1212
1213         data &= ~mask;
1214         data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1215         data |= mask & (rfr_level << 2);
1216         data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1217         msm_write(port, data, UART_MR1);
1218
1219         if (msm_port->is_uartdm) {
1220                 msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1221                 msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1222         }
1223
1224         ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1225                           msm_port->name, port);
1226         if (unlikely(ret))
1227                 goto err_irq;
1228
1229         return 0;
1230
1231 err_irq:
1232         if (msm_port->is_uartdm)
1233                 msm_release_dma(msm_port);
1234
1235         clk_disable_unprepare(msm_port->pclk);
1236         clk_disable_unprepare(msm_port->clk);
1237
1238         return ret;
1239 }
1240
1241 static void msm_shutdown(struct uart_port *port)
1242 {
1243         struct msm_port *msm_port = UART_TO_MSM(port);
1244
1245         msm_port->imr = 0;
1246         msm_write(port, 0, UART_IMR); /* disable interrupts */
1247
1248         if (msm_port->is_uartdm)
1249                 msm_release_dma(msm_port);
1250
1251         clk_disable_unprepare(msm_port->clk);
1252
1253         free_irq(port->irq, port);
1254 }
1255
1256 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1257                             struct ktermios *old)
1258 {
1259         struct msm_port *msm_port = UART_TO_MSM(port);
1260         struct msm_dma *dma = &msm_port->rx_dma;
1261         unsigned long flags;
1262         unsigned int baud, mr;
1263
1264         spin_lock_irqsave(&port->lock, flags);
1265
1266         if (dma->chan) /* Terminate if any */
1267                 msm_stop_dma(port, dma);
1268
1269         /* calculate and set baud rate */
1270         baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1271         baud = msm_set_baud_rate(port, baud, &flags);
1272         if (tty_termios_baud_rate(termios))
1273                 tty_termios_encode_baud_rate(termios, baud, baud);
1274
1275         /* calculate parity */
1276         mr = msm_read(port, UART_MR2);
1277         mr &= ~UART_MR2_PARITY_MODE;
1278         if (termios->c_cflag & PARENB) {
1279                 if (termios->c_cflag & PARODD)
1280                         mr |= UART_MR2_PARITY_MODE_ODD;
1281                 else if (termios->c_cflag & CMSPAR)
1282                         mr |= UART_MR2_PARITY_MODE_SPACE;
1283                 else
1284                         mr |= UART_MR2_PARITY_MODE_EVEN;
1285         }
1286
1287         /* calculate bits per char */
1288         mr &= ~UART_MR2_BITS_PER_CHAR;
1289         switch (termios->c_cflag & CSIZE) {
1290         case CS5:
1291                 mr |= UART_MR2_BITS_PER_CHAR_5;
1292                 break;
1293         case CS6:
1294                 mr |= UART_MR2_BITS_PER_CHAR_6;
1295                 break;
1296         case CS7:
1297                 mr |= UART_MR2_BITS_PER_CHAR_7;
1298                 break;
1299         case CS8:
1300         default:
1301                 mr |= UART_MR2_BITS_PER_CHAR_8;
1302                 break;
1303         }
1304
1305         /* calculate stop bits */
1306         mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1307         if (termios->c_cflag & CSTOPB)
1308                 mr |= UART_MR2_STOP_BIT_LEN_TWO;
1309         else
1310                 mr |= UART_MR2_STOP_BIT_LEN_ONE;
1311
1312         /* set parity, bits per char, and stop bit */
1313         msm_write(port, mr, UART_MR2);
1314
1315         /* calculate and set hardware flow control */
1316         mr = msm_read(port, UART_MR1);
1317         mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1318         if (termios->c_cflag & CRTSCTS) {
1319                 mr |= UART_MR1_CTS_CTL;
1320                 mr |= UART_MR1_RX_RDY_CTL;
1321         }
1322         msm_write(port, mr, UART_MR1);
1323
1324         /* Configure status bits to ignore based on termio flags. */
1325         port->read_status_mask = 0;
1326         if (termios->c_iflag & INPCK)
1327                 port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1328         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1329                 port->read_status_mask |= UART_SR_RX_BREAK;
1330
1331         uart_update_timeout(port, termios->c_cflag, baud);
1332
1333         /* Try to use DMA */
1334         msm_start_rx_dma(msm_port);
1335
1336         spin_unlock_irqrestore(&port->lock, flags);
1337 }
1338
1339 static const char *msm_type(struct uart_port *port)
1340 {
1341         return "MSM";
1342 }
1343
1344 static void msm_release_port(struct uart_port *port)
1345 {
1346         struct platform_device *pdev = to_platform_device(port->dev);
1347         struct resource *uart_resource;
1348         resource_size_t size;
1349
1350         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1351         if (unlikely(!uart_resource))
1352                 return;
1353         size = resource_size(uart_resource);
1354
1355         release_mem_region(port->mapbase, size);
1356         iounmap(port->membase);
1357         port->membase = NULL;
1358 }
1359
1360 static int msm_request_port(struct uart_port *port)
1361 {
1362         struct platform_device *pdev = to_platform_device(port->dev);
1363         struct resource *uart_resource;
1364         resource_size_t size;
1365         int ret;
1366
1367         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1368         if (unlikely(!uart_resource))
1369                 return -ENXIO;
1370
1371         size = resource_size(uart_resource);
1372
1373         if (!request_mem_region(port->mapbase, size, "msm_serial"))
1374                 return -EBUSY;
1375
1376         port->membase = ioremap(port->mapbase, size);
1377         if (!port->membase) {
1378                 ret = -EBUSY;
1379                 goto fail_release_port;
1380         }
1381
1382         return 0;
1383
1384 fail_release_port:
1385         release_mem_region(port->mapbase, size);
1386         return ret;
1387 }
1388
1389 static void msm_config_port(struct uart_port *port, int flags)
1390 {
1391         int ret;
1392
1393         if (flags & UART_CONFIG_TYPE) {
1394                 port->type = PORT_MSM;
1395                 ret = msm_request_port(port);
1396                 if (ret)
1397                         return;
1398         }
1399 }
1400
1401 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1402 {
1403         if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1404                 return -EINVAL;
1405         if (unlikely(port->irq != ser->irq))
1406                 return -EINVAL;
1407         return 0;
1408 }
1409
1410 static void msm_power(struct uart_port *port, unsigned int state,
1411                       unsigned int oldstate)
1412 {
1413         struct msm_port *msm_port = UART_TO_MSM(port);
1414
1415         switch (state) {
1416         case 0:
1417                 clk_prepare_enable(msm_port->clk);
1418                 clk_prepare_enable(msm_port->pclk);
1419                 break;
1420         case 3:
1421                 clk_disable_unprepare(msm_port->clk);
1422                 clk_disable_unprepare(msm_port->pclk);
1423                 break;
1424         default:
1425                 pr_err("msm_serial: Unknown PM state %d\n", state);
1426         }
1427 }
1428
1429 #ifdef CONFIG_CONSOLE_POLL
1430 static int msm_poll_get_char_single(struct uart_port *port)
1431 {
1432         struct msm_port *msm_port = UART_TO_MSM(port);
1433         unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1434
1435         if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1436                 return NO_POLL_CHAR;
1437
1438         return msm_read(port, rf_reg) & 0xff;
1439 }
1440
1441 static int msm_poll_get_char_dm(struct uart_port *port)
1442 {
1443         int c;
1444         static u32 slop;
1445         static int count;
1446         unsigned char *sp = (unsigned char *)&slop;
1447
1448         /* Check if a previous read had more than one char */
1449         if (count) {
1450                 c = sp[sizeof(slop) - count];
1451                 count--;
1452         /* Or if FIFO is empty */
1453         } else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1454                 /*
1455                  * If RX packing buffer has less than a word, force stale to
1456                  * push contents into RX FIFO
1457                  */
1458                 count = msm_read(port, UARTDM_RXFS);
1459                 count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1460                 if (count) {
1461                         msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1462                         slop = msm_read(port, UARTDM_RF);
1463                         c = sp[0];
1464                         count--;
1465                         msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1466                         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1467                         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1468                                   UART_CR);
1469                 } else {
1470                         c = NO_POLL_CHAR;
1471                 }
1472         /* FIFO has a word */
1473         } else {
1474                 slop = msm_read(port, UARTDM_RF);
1475                 c = sp[0];
1476                 count = sizeof(slop) - 1;
1477         }
1478
1479         return c;
1480 }
1481
1482 static int msm_poll_get_char(struct uart_port *port)
1483 {
1484         u32 imr;
1485         int c;
1486         struct msm_port *msm_port = UART_TO_MSM(port);
1487
1488         /* Disable all interrupts */
1489         imr = msm_read(port, UART_IMR);
1490         msm_write(port, 0, UART_IMR);
1491
1492         if (msm_port->is_uartdm)
1493                 c = msm_poll_get_char_dm(port);
1494         else
1495                 c = msm_poll_get_char_single(port);
1496
1497         /* Enable interrupts */
1498         msm_write(port, imr, UART_IMR);
1499
1500         return c;
1501 }
1502
1503 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1504 {
1505         u32 imr;
1506         struct msm_port *msm_port = UART_TO_MSM(port);
1507
1508         /* Disable all interrupts */
1509         imr = msm_read(port, UART_IMR);
1510         msm_write(port, 0, UART_IMR);
1511
1512         if (msm_port->is_uartdm)
1513                 msm_reset_dm_count(port, 1);
1514
1515         /* Wait until FIFO is empty */
1516         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1517                 cpu_relax();
1518
1519         /* Write a character */
1520         msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1521
1522         /* Wait until FIFO is empty */
1523         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1524                 cpu_relax();
1525
1526         /* Enable interrupts */
1527         msm_write(port, imr, UART_IMR);
1528 }
1529 #endif
1530
1531 static struct uart_ops msm_uart_pops = {
1532         .tx_empty = msm_tx_empty,
1533         .set_mctrl = msm_set_mctrl,
1534         .get_mctrl = msm_get_mctrl,
1535         .stop_tx = msm_stop_tx,
1536         .start_tx = msm_start_tx,
1537         .stop_rx = msm_stop_rx,
1538         .enable_ms = msm_enable_ms,
1539         .break_ctl = msm_break_ctl,
1540         .startup = msm_startup,
1541         .shutdown = msm_shutdown,
1542         .set_termios = msm_set_termios,
1543         .type = msm_type,
1544         .release_port = msm_release_port,
1545         .request_port = msm_request_port,
1546         .config_port = msm_config_port,
1547         .verify_port = msm_verify_port,
1548         .pm = msm_power,
1549 #ifdef CONFIG_CONSOLE_POLL
1550         .poll_get_char  = msm_poll_get_char,
1551         .poll_put_char  = msm_poll_put_char,
1552 #endif
1553 };
1554
1555 static struct msm_port msm_uart_ports[] = {
1556         {
1557                 .uart = {
1558                         .iotype = UPIO_MEM,
1559                         .ops = &msm_uart_pops,
1560                         .flags = UPF_BOOT_AUTOCONF,
1561                         .fifosize = 64,
1562                         .line = 0,
1563                 },
1564         },
1565         {
1566                 .uart = {
1567                         .iotype = UPIO_MEM,
1568                         .ops = &msm_uart_pops,
1569                         .flags = UPF_BOOT_AUTOCONF,
1570                         .fifosize = 64,
1571                         .line = 1,
1572                 },
1573         },
1574         {
1575                 .uart = {
1576                         .iotype = UPIO_MEM,
1577                         .ops = &msm_uart_pops,
1578                         .flags = UPF_BOOT_AUTOCONF,
1579                         .fifosize = 64,
1580                         .line = 2,
1581                 },
1582         },
1583 };
1584
1585 #define UART_NR ARRAY_SIZE(msm_uart_ports)
1586
1587 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1588 {
1589         return &msm_uart_ports[line].uart;
1590 }
1591
1592 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1593 static void __msm_console_write(struct uart_port *port, const char *s,
1594                                 unsigned int count, bool is_uartdm)
1595 {
1596         unsigned long flags;
1597         int i;
1598         int num_newlines = 0;
1599         bool replaced = false;
1600         void __iomem *tf;
1601         int locked = 1;
1602
1603         if (is_uartdm)
1604                 tf = port->membase + UARTDM_TF;
1605         else
1606                 tf = port->membase + UART_TF;
1607
1608         /* Account for newlines that will get a carriage return added */
1609         for (i = 0; i < count; i++)
1610                 if (s[i] == '\n')
1611                         num_newlines++;
1612         count += num_newlines;
1613
1614         local_irq_save(flags);
1615
1616         if (port->sysrq)
1617                 locked = 0;
1618         else if (oops_in_progress)
1619                 locked = spin_trylock(&port->lock);
1620         else
1621                 spin_lock(&port->lock);
1622
1623         if (is_uartdm)
1624                 msm_reset_dm_count(port, count);
1625
1626         i = 0;
1627         while (i < count) {
1628                 int j;
1629                 unsigned int num_chars;
1630                 char buf[4] = { 0 };
1631
1632                 if (is_uartdm)
1633                         num_chars = min(count - i, (unsigned int)sizeof(buf));
1634                 else
1635                         num_chars = 1;
1636
1637                 for (j = 0; j < num_chars; j++) {
1638                         char c = *s;
1639
1640                         if (c == '\n' && !replaced) {
1641                                 buf[j] = '\r';
1642                                 j++;
1643                                 replaced = true;
1644                         }
1645                         if (j < num_chars) {
1646                                 buf[j] = c;
1647                                 s++;
1648                                 replaced = false;
1649                         }
1650                 }
1651
1652                 while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1653                         cpu_relax();
1654
1655                 iowrite32_rep(tf, buf, 1);
1656                 i += num_chars;
1657         }
1658
1659         if (locked)
1660                 spin_unlock(&port->lock);
1661
1662         local_irq_restore(flags);
1663 }
1664
1665 static void msm_console_write(struct console *co, const char *s,
1666                               unsigned int count)
1667 {
1668         struct uart_port *port;
1669         struct msm_port *msm_port;
1670
1671         BUG_ON(co->index < 0 || co->index >= UART_NR);
1672
1673         port = msm_get_port_from_line(co->index);
1674         msm_port = UART_TO_MSM(port);
1675
1676         __msm_console_write(port, s, count, msm_port->is_uartdm);
1677 }
1678
1679 static int msm_console_setup(struct console *co, char *options)
1680 {
1681         struct uart_port *port;
1682         int baud = 115200;
1683         int bits = 8;
1684         int parity = 'n';
1685         int flow = 'n';
1686
1687         if (unlikely(co->index >= UART_NR || co->index < 0))
1688                 return -ENXIO;
1689
1690         port = msm_get_port_from_line(co->index);
1691
1692         if (unlikely(!port->membase))
1693                 return -ENXIO;
1694
1695         msm_init_clock(port);
1696
1697         if (options)
1698                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1699
1700         pr_info("msm_serial: console setup on port #%d\n", port->line);
1701
1702         return uart_set_options(port, co, baud, parity, bits, flow);
1703 }
1704
1705 static void
1706 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1707 {
1708         struct earlycon_device *dev = con->data;
1709
1710         __msm_console_write(&dev->port, s, n, false);
1711 }
1712
1713 static int __init
1714 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1715 {
1716         if (!device->port.membase)
1717                 return -ENODEV;
1718
1719         device->con->write = msm_serial_early_write;
1720         return 0;
1721 }
1722 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1723                     msm_serial_early_console_setup);
1724
1725 static void
1726 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1727 {
1728         struct earlycon_device *dev = con->data;
1729
1730         __msm_console_write(&dev->port, s, n, true);
1731 }
1732
1733 static int __init
1734 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1735                                   const char *opt)
1736 {
1737         if (!device->port.membase)
1738                 return -ENODEV;
1739
1740         device->con->write = msm_serial_early_write_dm;
1741         return 0;
1742 }
1743 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1744                     msm_serial_early_console_setup_dm);
1745
1746 static struct uart_driver msm_uart_driver;
1747
1748 static struct console msm_console = {
1749         .name = "ttyMSM",
1750         .write = msm_console_write,
1751         .device = uart_console_device,
1752         .setup = msm_console_setup,
1753         .flags = CON_PRINTBUFFER,
1754         .index = -1,
1755         .data = &msm_uart_driver,
1756 };
1757
1758 #define MSM_CONSOLE     (&msm_console)
1759
1760 #else
1761 #define MSM_CONSOLE     NULL
1762 #endif
1763
1764 static struct uart_driver msm_uart_driver = {
1765         .owner = THIS_MODULE,
1766         .driver_name = "msm_serial",
1767         .dev_name = "ttyMSM",
1768         .nr = UART_NR,
1769         .cons = MSM_CONSOLE,
1770 };
1771
1772 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1773
1774 static const struct of_device_id msm_uartdm_table[] = {
1775         { .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1776         { .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1777         { .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1778         { .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1779         { }
1780 };
1781
1782 static int msm_serial_probe(struct platform_device *pdev)
1783 {
1784         struct msm_port *msm_port;
1785         struct resource *resource;
1786         struct uart_port *port;
1787         const struct of_device_id *id;
1788         int irq, line;
1789
1790         if (pdev->dev.of_node)
1791                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1792         else
1793                 line = pdev->id;
1794
1795         if (line < 0)
1796                 line = atomic_inc_return(&msm_uart_next_id) - 1;
1797
1798         if (unlikely(line < 0 || line >= UART_NR))
1799                 return -ENXIO;
1800
1801         dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1802
1803         port = msm_get_port_from_line(line);
1804         port->dev = &pdev->dev;
1805         msm_port = UART_TO_MSM(port);
1806
1807         id = of_match_device(msm_uartdm_table, &pdev->dev);
1808         if (id)
1809                 msm_port->is_uartdm = (unsigned long)id->data;
1810         else
1811                 msm_port->is_uartdm = 0;
1812
1813         msm_port->clk = devm_clk_get(&pdev->dev, "core");
1814         if (IS_ERR(msm_port->clk))
1815                 return PTR_ERR(msm_port->clk);
1816
1817         if (msm_port->is_uartdm) {
1818                 msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1819                 if (IS_ERR(msm_port->pclk))
1820                         return PTR_ERR(msm_port->pclk);
1821         }
1822
1823         port->uartclk = clk_get_rate(msm_port->clk);
1824         dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1825
1826         resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1827         if (unlikely(!resource))
1828                 return -ENXIO;
1829         port->mapbase = resource->start;
1830
1831         irq = platform_get_irq(pdev, 0);
1832         if (unlikely(irq < 0))
1833                 return -ENXIO;
1834         port->irq = irq;
1835         port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MSM_CONSOLE);
1836
1837         platform_set_drvdata(pdev, port);
1838
1839         return uart_add_one_port(&msm_uart_driver, port);
1840 }
1841
1842 static int msm_serial_remove(struct platform_device *pdev)
1843 {
1844         struct uart_port *port = platform_get_drvdata(pdev);
1845
1846         uart_remove_one_port(&msm_uart_driver, port);
1847
1848         return 0;
1849 }
1850
1851 static const struct of_device_id msm_match_table[] = {
1852         { .compatible = "qcom,msm-uart" },
1853         { .compatible = "qcom,msm-uartdm" },
1854         {}
1855 };
1856 MODULE_DEVICE_TABLE(of, msm_match_table);
1857
1858 static int __maybe_unused msm_serial_suspend(struct device *dev)
1859 {
1860         struct msm_port *port = dev_get_drvdata(dev);
1861
1862         uart_suspend_port(&msm_uart_driver, &port->uart);
1863
1864         return 0;
1865 }
1866
1867 static int __maybe_unused msm_serial_resume(struct device *dev)
1868 {
1869         struct msm_port *port = dev_get_drvdata(dev);
1870
1871         uart_resume_port(&msm_uart_driver, &port->uart);
1872
1873         return 0;
1874 }
1875
1876 static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1877         SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1878 };
1879
1880 static struct platform_driver msm_platform_driver = {
1881         .remove = msm_serial_remove,
1882         .probe = msm_serial_probe,
1883         .driver = {
1884                 .name = "msm_serial",
1885                 .pm = &msm_serial_dev_pm_ops,
1886                 .of_match_table = msm_match_table,
1887         },
1888 };
1889
1890 static int __init msm_serial_init(void)
1891 {
1892         int ret;
1893
1894         ret = uart_register_driver(&msm_uart_driver);
1895         if (unlikely(ret))
1896                 return ret;
1897
1898         ret = platform_driver_register(&msm_platform_driver);
1899         if (unlikely(ret))
1900                 uart_unregister_driver(&msm_uart_driver);
1901
1902         pr_info("msm_serial: driver initialized\n");
1903
1904         return ret;
1905 }
1906
1907 static void __exit msm_serial_exit(void)
1908 {
1909         platform_driver_unregister(&msm_platform_driver);
1910         uart_unregister_driver(&msm_uart_driver);
1911 }
1912
1913 module_init(msm_serial_init);
1914 module_exit(msm_serial_exit);
1915
1916 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1917 MODULE_DESCRIPTION("Driver for msm7x serial device");
1918 MODULE_LICENSE("GPL");