drivers/tty/serial/msm_serial.c

   1 /*
   2  * Driver for msm7k serial device and console
   3  *
   4  * Copyright (C) 2007 Google, Inc.
   5  * Author: Robert Love <rlove@google.com>
   6  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   7  *
   8  * This software is licensed under the terms of the GNU General Public
   9  * License version 2, as published by the Free Software Foundation, and
  10  * may be copied, distributed, and modified under those terms.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  */
  17
  18 #if defined(CONFIG_SERIAL_MSM_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
  19 # define SUPPORT_SYSRQ
  20 #endif
  21
  22 #include <linux/kernel.h>
  23 #include <linux/atomic.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/dmaengine.h>
  26 #include <linux/module.h>
  27 #include <linux/io.h>
  28 #include <linux/ioport.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/init.h>
  31 #include <linux/console.h>
  32 #include <linux/tty.h>
  33 #include <linux/tty_flip.h>
  34 #include <linux/serial_core.h>
  35 #include <linux/slab.h>
  36 #include <linux/clk.h>
  37 #include <linux/platform_device.h>
  38 #include <linux/delay.h>
  39 #include <linux/of.h>
  40 #include <linux/of_device.h>
  41 #include <linux/wait.h>
  42
  43 #define UART_MR1                        0x0000
  44
  45 #define UART_MR1_AUTO_RFR_LEVEL0        0x3F
  46 #define UART_MR1_AUTO_RFR_LEVEL1        0x3FF00
  47 #define UART_DM_MR1_AUTO_RFR_LEVEL1     0xFFFFFF00
  48 #define UART_MR1_RX_RDY_CTL             BIT(7)
  49 #define UART_MR1_CTS_CTL                BIT(6)
  50
  51 #define UART_MR2                        0x0004
  52 #define UART_MR2_ERROR_MODE             BIT(6)
  53 #define UART_MR2_BITS_PER_CHAR          0x30
  54 #define UART_MR2_BITS_PER_CHAR_5        (0x0 << 4)
  55 #define UART_MR2_BITS_PER_CHAR_6        (0x1 << 4)
  56 #define UART_MR2_BITS_PER_CHAR_7        (0x2 << 4)
  57 #define UART_MR2_BITS_PER_CHAR_8        (0x3 << 4)
  58 #define UART_MR2_STOP_BIT_LEN_ONE       (0x1 << 2)
  59 #define UART_MR2_STOP_BIT_LEN_TWO       (0x3 << 2)
  60 #define UART_MR2_PARITY_MODE_NONE       0x0
  61 #define UART_MR2_PARITY_MODE_ODD        0x1
  62 #define UART_MR2_PARITY_MODE_EVEN       0x2
  63 #define UART_MR2_PARITY_MODE_SPACE      0x3
  64 #define UART_MR2_PARITY_MODE            0x3
  65
  66 #define UART_CSR                        0x0008
  67
  68 #define UART_TF                         0x000C
  69 #define UARTDM_TF                       0x0070
  70
  71 #define UART_CR                         0x0010
  72 #define UART_CR_CMD_NULL                (0 << 4)
  73 #define UART_CR_CMD_RESET_RX            (1 << 4)
  74 #define UART_CR_CMD_RESET_TX            (2 << 4)
  75 #define UART_CR_CMD_RESET_ERR           (3 << 4)
  76 #define UART_CR_CMD_RESET_BREAK_INT     (4 << 4)
  77 #define UART_CR_CMD_START_BREAK         (5 << 4)
  78 #define UART_CR_CMD_STOP_BREAK          (6 << 4)
  79 #define UART_CR_CMD_RESET_CTS           (7 << 4)
  80 #define UART_CR_CMD_RESET_STALE_INT     (8 << 4)
  81 #define UART_CR_CMD_PACKET_MODE         (9 << 4)
  82 #define UART_CR_CMD_MODE_RESET          (12 << 4)
  83 #define UART_CR_CMD_SET_RFR             (13 << 4)
  84 #define UART_CR_CMD_RESET_RFR           (14 << 4)
  85 #define UART_CR_CMD_PROTECTION_EN       (16 << 4)
  86 #define UART_CR_CMD_STALE_EVENT_DISABLE (6 << 8)
  87 #define UART_CR_CMD_STALE_EVENT_ENABLE  (80 << 4)
  88 #define UART_CR_CMD_FORCE_STALE         (4 << 8)
  89 #define UART_CR_CMD_RESET_TX_READY      (3 << 8)
  90 #define UART_CR_TX_DISABLE              BIT(3)
  91 #define UART_CR_TX_ENABLE               BIT(2)
  92 #define UART_CR_RX_DISABLE              BIT(1)
  93 #define UART_CR_RX_ENABLE               BIT(0)
  94 #define UART_CR_CMD_RESET_RXBREAK_START ((1 << 11) | (2 << 4))
  95
  96 #define UART_IMR                        0x0014
  97 #define UART_IMR_TXLEV                  BIT(0)
  98 #define UART_IMR_RXSTALE                BIT(3)
  99 #define UART_IMR_RXLEV                  BIT(4)
 100 #define UART_IMR_DELTA_CTS              BIT(5)
 101 #define UART_IMR_CURRENT_CTS            BIT(6)
 102 #define UART_IMR_RXBREAK_START          BIT(10)
 103
 104 #define UART_IPR_RXSTALE_LAST           0x20
 105 #define UART_IPR_STALE_LSB              0x1F
 106 #define UART_IPR_STALE_TIMEOUT_MSB      0x3FF80
 107 #define UART_DM_IPR_STALE_TIMEOUT_MSB   0xFFFFFF80
 108
 109 #define UART_IPR                        0x0018
 110 #define UART_TFWR                       0x001C
 111 #define UART_RFWR                       0x0020
 112 #define UART_HCR                        0x0024
 113
 114 #define UART_MREG                       0x0028
 115 #define UART_NREG                       0x002C
 116 #define UART_DREG                       0x0030
 117 #define UART_MNDREG                     0x0034
 118 #define UART_IRDA                       0x0038
 119 #define UART_MISR_MODE                  0x0040
 120 #define UART_MISR_RESET                 0x0044
 121 #define UART_MISR_EXPORT                0x0048
 122 #define UART_MISR_VAL                   0x004C
 123 #define UART_TEST_CTRL                  0x0050
 124
 125 #define UART_SR                         0x0008
 126 #define UART_SR_HUNT_CHAR               BIT(7)
 127 #define UART_SR_RX_BREAK                BIT(6)
 128 #define UART_SR_PAR_FRAME_ERR           BIT(5)
 129 #define UART_SR_OVERRUN                 BIT(4)
 130 #define UART_SR_TX_EMPTY                BIT(3)
 131 #define UART_SR_TX_READY                BIT(2)
 132 #define UART_SR_RX_FULL                 BIT(1)
 133 #define UART_SR_RX_READY                BIT(0)
 134
 135 #define UART_RF                         0x000C
 136 #define UARTDM_RF                       0x0070
 137 #define UART_MISR                       0x0010
 138 #define UART_ISR                        0x0014
 139 #define UART_ISR_TX_READY               BIT(7)
 140
 141 #define UARTDM_RXFS                     0x50
 142 #define UARTDM_RXFS_BUF_SHIFT           0x7
 143 #define UARTDM_RXFS_BUF_MASK            0x7
 144
 145 #define UARTDM_DMEN                     0x3C
 146 #define UARTDM_DMEN_RX_SC_ENABLE        BIT(5)
 147 #define UARTDM_DMEN_TX_SC_ENABLE        BIT(4)
 148
 149 #define UARTDM_DMEN_TX_BAM_ENABLE       BIT(2)  /* UARTDM_1P4 */
 150 #define UARTDM_DMEN_TX_DM_ENABLE        BIT(0)  /* < UARTDM_1P4 */
 151
 152 #define UARTDM_DMEN_RX_BAM_ENABLE       BIT(3)  /* UARTDM_1P4 */
 153 #define UARTDM_DMEN_RX_DM_ENABLE        BIT(1)  /* < UARTDM_1P4 */
 154
 155 #define UARTDM_DMRX                     0x34
 156 #define UARTDM_NCF_TX                   0x40
 157 #define UARTDM_RX_TOTAL_SNAP            0x38
 158
 159 #define UARTDM_BURST_SIZE               16   /* in bytes */
 160 #define UARTDM_TX_AIGN(x)               ((x) & ~0x3) /* valid for > 1p3 */
 161 #define UARTDM_TX_MAX                   256   /* in bytes, valid for <= 1p3 */
 162 #define UARTDM_RX_SIZE                  (UART_XMIT_SIZE / 4)
 163
 164 enum {
 165         UARTDM_1P1 = 1,
 166         UARTDM_1P2,
 167         UARTDM_1P3,
 168         UARTDM_1P4,
 169 };
 170
 171 struct msm_dma {
 172         struct dma_chan         *chan;
 173         enum dma_data_direction dir;
 174         dma_addr_t              phys;
 175         unsigned char           *virt;
 176         dma_cookie_t            cookie;
 177         u32                     enable_bit;
 178         unsigned int            count;
 179         struct dma_async_tx_descriptor  *desc;
 180 };
 181
 182 struct msm_port {
 183         struct uart_port        uart;
 184         char                    name[16];
 185         struct clk              *clk;
 186         struct clk              *pclk;
 187         unsigned int            imr;
 188         int                     is_uartdm;
 189         unsigned int            old_snap_state;
 190         bool                    break_detected;
 191         struct msm_dma          tx_dma;
 192         struct msm_dma          rx_dma;
 193 };
 194
 195 #define UART_TO_MSM(uart_port)  container_of(uart_port, struct msm_port, uart)
 196
 197 static
 198 void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 199 {
 200         writel_relaxed(val, port->membase + off);
 201 }
 202
 203 static
 204 unsigned int msm_read(struct uart_port *port, unsigned int off)
 205 {
 206         return readl_relaxed(port->membase + off);
 207 }
 208
 209 /*
 210  * Setup the MND registers to use the TCXO clock.
 211  */
 212 static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 213 {
 214         msm_write(port, 0x06, UART_MREG);
 215         msm_write(port, 0xF1, UART_NREG);
 216         msm_write(port, 0x0F, UART_DREG);
 217         msm_write(port, 0x1A, UART_MNDREG);
 218         port->uartclk = 1843200;
 219 }
 220
 221 /*
 222  * Setup the MND registers to use the TCXO clock divided by 4.
 223  */
 224 static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 225 {
 226         msm_write(port, 0x18, UART_MREG);
 227         msm_write(port, 0xF6, UART_NREG);
 228         msm_write(port, 0x0F, UART_DREG);
 229         msm_write(port, 0x0A, UART_MNDREG);
 230         port->uartclk = 1843200;
 231 }
 232
 233 static void msm_serial_set_mnd_regs(struct uart_port *port)
 234 {
 235         struct msm_port *msm_port = UART_TO_MSM(port);
 236
 237         /*
 238          * These registers don't exist so we change the clk input rate
 239          * on uartdm hardware instead
 240          */
 241         if (msm_port->is_uartdm)
 242                 return;
 243
 244         if (port->uartclk == 19200000)
 245                 msm_serial_set_mnd_regs_tcxo(port);
 246         else if (port->uartclk == 4800000)
 247                 msm_serial_set_mnd_regs_tcxoby4(port);
 248 }
 249
 250 static void msm_handle_tx(struct uart_port *port);
 251 static void msm_start_rx_dma(struct msm_port *msm_port);
 252
 253 static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 254 {
 255         struct device *dev = port->dev;
 256         unsigned int mapped;
 257         u32 val;
 258
 259         mapped = dma->count;
 260         dma->count = 0;
 261
 262         dmaengine_terminate_all(dma->chan);
 263
 264         /*
 265          * DMA Stall happens if enqueue and flush command happens concurrently.
 266          * For example before changing the baud rate/protocol configuration and
 267          * sending flush command to ADM, disable the channel of UARTDM.
 268          * Note: should not reset the receiver here immediately as it is not
 269          * suggested to do disable/reset or reset/disable at the same time.
 270          */
 271         val = msm_read(port, UARTDM_DMEN);
 272         val &= ~dma->enable_bit;
 273         msm_write(port, val, UARTDM_DMEN);
 274
 275         if (mapped)
 276                 dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 277 }
 278
 279 static void msm_release_dma(struct msm_port *msm_port)
 280 {
 281         struct msm_dma *dma;
 282
 283         dma = &msm_port->tx_dma;
 284         if (dma->chan) {
 285                 msm_stop_dma(&msm_port->uart, dma);
 286                 dma_release_channel(dma->chan);
 287         }
 288
 289         memset(dma, 0, sizeof(*dma));
 290
 291         dma = &msm_port->rx_dma;
 292         if (dma->chan) {
 293                 msm_stop_dma(&msm_port->uart, dma);
 294                 dma_release_channel(dma->chan);
 295                 kfree(dma->virt);
 296         }
 297
 298         memset(dma, 0, sizeof(*dma));
 299 }
 300
 301 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 302 {
 303         struct device *dev = msm_port->uart.dev;
 304         struct dma_slave_config conf;
 305         struct msm_dma *dma;
 306         u32 crci = 0;
 307         int ret;
 308
 309         dma = &msm_port->tx_dma;
 310
 311         /* allocate DMA resources, if available */
 312         dma->chan = dma_request_slave_channel_reason(dev, "tx");
 313         if (IS_ERR(dma->chan))
 314                 goto no_tx;
 315
 316         of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 317
 318         memset(&conf, 0, sizeof(conf));
 319         conf.direction = DMA_MEM_TO_DEV;
 320         conf.device_fc = true;
 321         conf.dst_addr = base + UARTDM_TF;
 322         conf.dst_maxburst = UARTDM_BURST_SIZE;
 323         conf.slave_id = crci;
 324
 325         ret = dmaengine_slave_config(dma->chan, &conf);
 326         if (ret)
 327                 goto rel_tx;
 328
 329         dma->dir = DMA_TO_DEVICE;
 330
 331         if (msm_port->is_uartdm < UARTDM_1P4)
 332                 dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 333         else
 334                 dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 335
 336         return;
 337
 338 rel_tx:
 339         dma_release_channel(dma->chan);
 340 no_tx:
 341         memset(dma, 0, sizeof(*dma));
 342 }
 343
 344 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 345 {
 346         struct device *dev = msm_port->uart.dev;
 347         struct dma_slave_config conf;
 348         struct msm_dma *dma;
 349         u32 crci = 0;
 350         int ret;
 351
 352         dma = &msm_port->rx_dma;
 353
 354         /* allocate DMA resources, if available */
 355         dma->chan = dma_request_slave_channel_reason(dev, "rx");
 356         if (IS_ERR(dma->chan))
 357                 goto no_rx;
 358
 359         of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 360
 361         dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 362         if (!dma->virt)
 363                 goto rel_rx;
 364
 365         memset(&conf, 0, sizeof(conf));
 366         conf.direction = DMA_DEV_TO_MEM;
 367         conf.device_fc = true;
 368         conf.src_addr = base + UARTDM_RF;
 369         conf.src_maxburst = UARTDM_BURST_SIZE;
 370         conf.slave_id = crci;
 371
 372         ret = dmaengine_slave_config(dma->chan, &conf);
 373         if (ret)
 374                 goto err;
 375
 376         dma->dir = DMA_FROM_DEVICE;
 377
 378         if (msm_port->is_uartdm < UARTDM_1P4)
 379                 dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 380         else
 381                 dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 382
 383         return;
 384 err:
 385         kfree(dma->virt);
 386 rel_rx:
 387         dma_release_channel(dma->chan);
 388 no_rx:
 389         memset(dma, 0, sizeof(*dma));
 390 }
 391
 392 static inline void msm_wait_for_xmitr(struct uart_port *port)
 393 {
 394         unsigned int timeout = 500000;
 395
 396         while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 397                 if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 398                         break;
 399                 udelay(1);
 400                 if (!timeout--)
 401                         break;
 402         }
 403         msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 404 }
 405
 406 static void msm_stop_tx(struct uart_port *port)
 407 {
 408         struct msm_port *msm_port = UART_TO_MSM(port);
 409
 410         msm_port->imr &= ~UART_IMR_TXLEV;
 411         msm_write(port, msm_port->imr, UART_IMR);
 412 }
 413
 414 static void msm_start_tx(struct uart_port *port)
 415 {
 416         struct msm_port *msm_port = UART_TO_MSM(port);
 417         struct msm_dma *dma = &msm_port->tx_dma;
 418
 419         /* Already started in DMA mode */
 420         if (dma->count)
 421                 return;
 422
 423         msm_port->imr |= UART_IMR_TXLEV;
 424         msm_write(port, msm_port->imr, UART_IMR);
 425 }
 426
 427 static void msm_reset_dm_count(struct uart_port *port, int count)
 428 {
 429         msm_wait_for_xmitr(port);
 430         msm_write(port, count, UARTDM_NCF_TX);
 431         msm_read(port, UARTDM_NCF_TX);
 432 }
 433
 434 static void msm_complete_tx_dma(void *args)
 435 {
 436         struct msm_port *msm_port = args;
 437         struct uart_port *port = &msm_port->uart;
 438         struct circ_buf *xmit = &port->state->xmit;
 439         struct msm_dma *dma = &msm_port->tx_dma;
 440         struct dma_tx_state state;
 441         enum dma_status status;
 442         unsigned long flags;
 443         unsigned int count;
 444         u32 val;
 445
 446         spin_lock_irqsave(&port->lock, flags);
 447
 448         /* Already stopped */
 449         if (!dma->count)
 450                 goto done;
 451
 452         status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
 453
 454         dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 455
 456         val = msm_read(port, UARTDM_DMEN);
 457         val &= ~dma->enable_bit;
 458         msm_write(port, val, UARTDM_DMEN);
 459
 460         if (msm_port->is_uartdm > UARTDM_1P3) {
 461                 msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 462                 msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 463         }
 464
 465         count = dma->count - state.residue;
 466         port->icount.tx += count;
 467         dma->count = 0;
 468
 469         xmit->tail += count;
 470         xmit->tail &= UART_XMIT_SIZE - 1;
 471
 472         /* Restore "Tx FIFO below watermark" interrupt */
 473         msm_port->imr |= UART_IMR_TXLEV;
 474         msm_write(port, msm_port->imr, UART_IMR);
 475
 476         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 477                 uart_write_wakeup(port);
 478
 479         msm_handle_tx(port);
 480 done:
 481         spin_unlock_irqrestore(&port->lock, flags);
 482 }
 483
 484 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 485 {
 486         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 487         struct uart_port *port = &msm_port->uart;
 488         struct msm_dma *dma = &msm_port->tx_dma;
 489         void *cpu_addr;
 490         int ret;
 491         u32 val;
 492
 493         cpu_addr = &xmit->buf[xmit->tail];
 494
 495         dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 496         ret = dma_mapping_error(port->dev, dma->phys);
 497         if (ret)
 498                 return ret;
 499
 500         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 501                                                 count, DMA_MEM_TO_DEV,
 502                                                 DMA_PREP_INTERRUPT |
 503                                                 DMA_PREP_FENCE);
 504         if (!dma->desc) {
 505                 ret = -EIO;
 506                 goto unmap;
 507         }
 508
 509         dma->desc->callback = msm_complete_tx_dma;
 510         dma->desc->callback_param = msm_port;
 511
 512         dma->cookie = dmaengine_submit(dma->desc);
 513         ret = dma_submit_error(dma->cookie);
 514         if (ret)
 515                 goto unmap;
 516
 517         /*
 518          * Using DMA complete for Tx FIFO reload, no need for
 519          * "Tx FIFO below watermark" one, disable it
 520          */
 521         msm_port->imr &= ~UART_IMR_TXLEV;
 522         msm_write(port, msm_port->imr, UART_IMR);
 523
 524         dma->count = count;
 525
 526         val = msm_read(port, UARTDM_DMEN);
 527         val |= dma->enable_bit;
 528
 529         if (msm_port->is_uartdm < UARTDM_1P4)
 530                 msm_write(port, val, UARTDM_DMEN);
 531
 532         msm_reset_dm_count(port, count);
 533
 534         if (msm_port->is_uartdm > UARTDM_1P3)
 535                 msm_write(port, val, UARTDM_DMEN);
 536
 537         dma_async_issue_pending(dma->chan);
 538         return 0;
 539 unmap:
 540         dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 541         return ret;
 542 }
 543
 544 static void msm_complete_rx_dma(void *args)
 545 {
 546         struct msm_port *msm_port = args;
 547         struct uart_port *port = &msm_port->uart;
 548         struct tty_port *tport = &port->state->port;
 549         struct msm_dma *dma = &msm_port->rx_dma;
 550         int count = 0, i, sysrq;
 551         unsigned long flags;
 552         u32 val;
 553
 554         spin_lock_irqsave(&port->lock, flags);
 555
 556         /* Already stopped */
 557         if (!dma->count)
 558                 goto done;
 559
 560         val = msm_read(port, UARTDM_DMEN);
 561         val &= ~dma->enable_bit;
 562         msm_write(port, val, UARTDM_DMEN);
 563
 564         if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 565                 port->icount.overrun++;
 566                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 567                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 568         }
 569
 570         count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 571
 572         port->icount.rx += count;
 573
 574         dma->count = 0;
 575
 576         dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 577
 578         for (i = 0; i < count; i++) {
 579                 char flag = TTY_NORMAL;
 580
 581                 if (msm_port->break_detected && dma->virt[i] == 0) {
 582                         port->icount.brk++;
 583                         flag = TTY_BREAK;
 584                         msm_port->break_detected = false;
 585                         if (uart_handle_break(port))
 586                                 continue;
 587                 }
 588
 589                 if (!(port->read_status_mask & UART_SR_RX_BREAK))
 590                         flag = TTY_NORMAL;
 591
 592                 spin_unlock_irqrestore(&port->lock, flags);
 593                 sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 594                 spin_lock_irqsave(&port->lock, flags);
 595                 if (!sysrq)
 596                         tty_insert_flip_char(tport, dma->virt[i], flag);
 597         }
 598
 599         msm_start_rx_dma(msm_port);
 600 done:
 601         spin_unlock_irqrestore(&port->lock, flags);
 602
 603         if (count)
 604                 tty_flip_buffer_push(tport);
 605 }
 606
 607 static void msm_start_rx_dma(struct msm_port *msm_port)
 608 {
 609         struct msm_dma *dma = &msm_port->rx_dma;
 610         struct uart_port *uart = &msm_port->uart;
 611         u32 val;
 612         int ret;
 613
 614         if (IS_ENABLED(CONFIG_CONSOLE_POLL))
 615                 return;
 616
 617         if (!dma->chan)
 618                 return;
 619
 620         dma->phys = dma_map_single(uart->dev, dma->virt,
 621                                    UARTDM_RX_SIZE, dma->dir);
 622         ret = dma_mapping_error(uart->dev, dma->phys);
 623         if (ret)
 624                 return;
 625
 626         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 627                                                 UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 628                                                 DMA_PREP_INTERRUPT);
 629         if (!dma->desc)
 630                 goto unmap;
 631
 632         dma->desc->callback = msm_complete_rx_dma;
 633         dma->desc->callback_param = msm_port;
 634
 635         dma->cookie = dmaengine_submit(dma->desc);
 636         ret = dma_submit_error(dma->cookie);
 637         if (ret)
 638                 goto unmap;
 639         /*
 640          * Using DMA for FIFO off-load, no need for "Rx FIFO over
 641          * watermark" or "stale" interrupts, disable them
 642          */
 643         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 644
 645         /*
 646          * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 647          * we need RXSTALE to flush input DMA fifo to memory
 648          */
 649         if (msm_port->is_uartdm < UARTDM_1P4)
 650                 msm_port->imr |= UART_IMR_RXSTALE;
 651
 652         msm_write(uart, msm_port->imr, UART_IMR);
 653
 654         dma->count = UARTDM_RX_SIZE;
 655
 656         dma_async_issue_pending(dma->chan);
 657
 658         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 659         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 660
 661         val = msm_read(uart, UARTDM_DMEN);
 662         val |= dma->enable_bit;
 663
 664         if (msm_port->is_uartdm < UARTDM_1P4)
 665                 msm_write(uart, val, UARTDM_DMEN);
 666
 667         msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 668
 669         if (msm_port->is_uartdm > UARTDM_1P3)
 670                 msm_write(uart, val, UARTDM_DMEN);
 671
 672         return;
 673 unmap:
 674         dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 675 }
 676
 677 static void msm_stop_rx(struct uart_port *port)
 678 {
 679         struct msm_port *msm_port = UART_TO_MSM(port);
 680         struct msm_dma *dma = &msm_port->rx_dma;
 681
 682         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 683         msm_write(port, msm_port->imr, UART_IMR);
 684
 685         if (dma->chan)
 686                 msm_stop_dma(port, dma);
 687 }
 688
 689 static void msm_enable_ms(struct uart_port *port)
 690 {
 691         struct msm_port *msm_port = UART_TO_MSM(port);
 692
 693         msm_port->imr |= UART_IMR_DELTA_CTS;
 694         msm_write(port, msm_port->imr, UART_IMR);
 695 }
 696
 697 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 698 {
 699         struct tty_port *tport = &port->state->port;
 700         unsigned int sr;
 701         int count = 0;
 702         struct msm_port *msm_port = UART_TO_MSM(port);
 703
 704         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 705                 port->icount.overrun++;
 706                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 707                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 708         }
 709
 710         if (misr & UART_IMR_RXSTALE) {
 711                 count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 712                         msm_port->old_snap_state;
 713                 msm_port->old_snap_state = 0;
 714         } else {
 715                 count = 4 * (msm_read(port, UART_RFWR));
 716                 msm_port->old_snap_state += count;
 717         }
 718
 719         /* TODO: Precise error reporting */
 720
 721         port->icount.rx += count;
 722
 723         while (count > 0) {
 724                 unsigned char buf[4];
 725                 int sysrq, r_count, i;
 726
 727                 sr = msm_read(port, UART_SR);
 728                 if ((sr & UART_SR_RX_READY) == 0) {
 729                         msm_port->old_snap_state -= count;
 730                         break;
 731                 }
 732
 733                 ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 734                 r_count = min_t(int, count, sizeof(buf));
 735
 736                 for (i = 0; i < r_count; i++) {
 737                         char flag = TTY_NORMAL;
 738
 739                         if (msm_port->break_detected && buf[i] == 0) {
 740                                 port->icount.brk++;
 741                                 flag = TTY_BREAK;
 742                                 msm_port->break_detected = false;
 743                                 if (uart_handle_break(port))
 744                                         continue;
 745                         }
 746
 747                         if (!(port->read_status_mask & UART_SR_RX_BREAK))
 748                                 flag = TTY_NORMAL;
 749
 750                         spin_unlock(&port->lock);
 751                         sysrq = uart_handle_sysrq_char(port, buf[i]);
 752                         spin_lock(&port->lock);
 753                         if (!sysrq)
 754                                 tty_insert_flip_char(tport, buf[i], flag);
 755                 }
 756                 count -= r_count;
 757         }
 758
 759         spin_unlock(&port->lock);
 760         tty_flip_buffer_push(tport);
 761         spin_lock(&port->lock);
 762
 763         if (misr & (UART_IMR_RXSTALE))
 764                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 765         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 766         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 767
 768         /* Try to use DMA */
 769         msm_start_rx_dma(msm_port);
 770 }
 771
 772 static void msm_handle_rx(struct uart_port *port)
 773 {
 774         struct tty_port *tport = &port->state->port;
 775         unsigned int sr;
 776
 777         /*
 778          * Handle overrun. My understanding of the hardware is that overrun
 779          * is not tied to the RX buffer, so we handle the case out of band.
 780          */
 781         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 782                 port->icount.overrun++;
 783                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 784                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 785         }
 786
 787         /* and now the main RX loop */
 788         while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 789                 unsigned int c;
 790                 char flag = TTY_NORMAL;
 791                 int sysrq;
 792
 793                 c = msm_read(port, UART_RF);
 794
 795                 if (sr & UART_SR_RX_BREAK) {
 796                         port->icount.brk++;
 797                         if (uart_handle_break(port))
 798                                 continue;
 799                 } else if (sr & UART_SR_PAR_FRAME_ERR) {
 800                         port->icount.frame++;
 801                 } else {
 802                         port->icount.rx++;
 803                 }
 804
 805                 /* Mask conditions we're ignorning. */
 806                 sr &= port->read_status_mask;
 807
 808                 if (sr & UART_SR_RX_BREAK)
 809                         flag = TTY_BREAK;
 810                 else if (sr & UART_SR_PAR_FRAME_ERR)
 811                         flag = TTY_FRAME;
 812
 813                 spin_unlock(&port->lock);
 814                 sysrq = uart_handle_sysrq_char(port, c);
 815                 spin_lock(&port->lock);
 816                 if (!sysrq)
 817                         tty_insert_flip_char(tport, c, flag);
 818         }
 819
 820         spin_unlock(&port->lock);
 821         tty_flip_buffer_push(tport);
 822         spin_lock(&port->lock);
 823 }
 824
 825 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 826 {
 827         struct circ_buf *xmit = &port->state->xmit;
 828         struct msm_port *msm_port = UART_TO_MSM(port);
 829         unsigned int num_chars;
 830         unsigned int tf_pointer = 0;
 831         void __iomem *tf;
 832
 833         if (msm_port->is_uartdm)
 834                 tf = port->membase + UARTDM_TF;
 835         else
 836                 tf = port->membase + UART_TF;
 837
 838         if (tx_count && msm_port->is_uartdm)
 839                 msm_reset_dm_count(port, tx_count);
 840
 841         while (tf_pointer < tx_count) {
 842                 int i;
 843                 char buf[4] = { 0 };
 844
 845                 if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 846                         break;
 847
 848                 if (msm_port->is_uartdm)
 849                         num_chars = min(tx_count - tf_pointer,
 850                                         (unsigned int)sizeof(buf));
 851                 else
 852                         num_chars = 1;
 853
 854                 for (i = 0; i < num_chars; i++) {
 855                         buf[i] = xmit->buf[xmit->tail + i];
 856                         port->icount.tx++;
 857                 }
 858
 859                 iowrite32_rep(tf, buf, 1);
 860                 xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 861                 tf_pointer += num_chars;
 862         }
 863
 864         /* disable tx interrupts if nothing more to send */
 865         if (uart_circ_empty(xmit))
 866                 msm_stop_tx(port);
 867
 868         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 869                 uart_write_wakeup(port);
 870 }
 871
 872 static void msm_handle_tx(struct uart_port *port)
 873 {
 874         struct msm_port *msm_port = UART_TO_MSM(port);
 875         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 876         struct msm_dma *dma = &msm_port->tx_dma;
 877         unsigned int pio_count, dma_count, dma_min;
 878         char buf[4] = { 0 };
 879         void __iomem *tf;
 880         int err = 0;
 881
 882         if (port->x_char) {
 883                 if (msm_port->is_uartdm)
 884                         tf = port->membase + UARTDM_TF;
 885                 else
 886                         tf = port->membase + UART_TF;
 887
 888                 buf[0] = port->x_char;
 889
 890                 if (msm_port->is_uartdm)
 891                         msm_reset_dm_count(port, 1);
 892
 893                 iowrite32_rep(tf, buf, 1);
 894                 port->icount.tx++;
 895                 port->x_char = 0;
 896                 return;
 897         }
 898
 899         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 900                 msm_stop_tx(port);
 901                 return;
 902         }
 903
 904         pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 905         dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 906
 907         dma_min = 1;    /* Always DMA */
 908         if (msm_port->is_uartdm > UARTDM_1P3) {
 909                 dma_count = UARTDM_TX_AIGN(dma_count);
 910                 dma_min = UARTDM_BURST_SIZE;
 911         } else {
 912                 if (dma_count > UARTDM_TX_MAX)
 913                         dma_count = UARTDM_TX_MAX;
 914         }
 915
 916         if (pio_count > port->fifosize)
 917                 pio_count = port->fifosize;
 918
 919         if (!dma->chan || dma_count < dma_min)
 920                 msm_handle_tx_pio(port, pio_count);
 921         else
 922                 err = msm_handle_tx_dma(msm_port, dma_count);
 923
 924         if (err)        /* fall back to PIO mode */
 925                 msm_handle_tx_pio(port, pio_count);
 926 }
 927
 928 static void msm_handle_delta_cts(struct uart_port *port)
 929 {
 930         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 931         port->icount.cts++;
 932         wake_up_interruptible(&port->state->port.delta_msr_wait);
 933 }
 934
 935 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 936 {
 937         struct uart_port *port = dev_id;
 938         struct msm_port *msm_port = UART_TO_MSM(port);
 939         struct msm_dma *dma = &msm_port->rx_dma;
 940         unsigned long flags;
 941         unsigned int misr;
 942         u32 val;
 943
 944         spin_lock_irqsave(&port->lock, flags);
 945         misr = msm_read(port, UART_MISR);
 946         msm_write(port, 0, UART_IMR); /* disable interrupt */
 947
 948         if (misr & UART_IMR_RXBREAK_START) {
 949                 msm_port->break_detected = true;
 950                 msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 951         }
 952
 953         if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 954                 if (dma->count) {
 955                         val = UART_CR_CMD_STALE_EVENT_DISABLE;
 956                         msm_write(port, val, UART_CR);
 957                         val = UART_CR_CMD_RESET_STALE_INT;
 958                         msm_write(port, val, UART_CR);
 959                         /*
 960                          * Flush DMA input fifo to memory, this will also
 961                          * trigger DMA RX completion
 962                          */
 963                         dmaengine_terminate_all(dma->chan);
 964                 } else if (msm_port->is_uartdm) {
 965                         msm_handle_rx_dm(port, misr);
 966                 } else {
 967                         msm_handle_rx(port);
 968                 }
 969         }
 970         if (misr & UART_IMR_TXLEV)
 971                 msm_handle_tx(port);
 972         if (misr & UART_IMR_DELTA_CTS)
 973                 msm_handle_delta_cts(port);
 974
 975         msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 976         spin_unlock_irqrestore(&port->lock, flags);
 977
 978         return IRQ_HANDLED;
 979 }
 980
 981 static unsigned int msm_tx_empty(struct uart_port *port)
 982 {
 983         return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 984 }
 985
 986 static unsigned int msm_get_mctrl(struct uart_port *port)
 987 {
 988         return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
 989 }
 990
 991 static void msm_reset(struct uart_port *port)
 992 {
 993         struct msm_port *msm_port = UART_TO_MSM(port);
 994         unsigned int mr;
 995
 996         /* reset everything */
 997         msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
 998         msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 999         msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
1000         msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
1001         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
1002         msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1003         mr = msm_read(port, UART_MR1);
1004         mr &= ~UART_MR1_RX_RDY_CTL;
1005         msm_write(port, mr, UART_MR1);
1006
1007         /* Disable DM modes */
1008         if (msm_port->is_uartdm)
1009                 msm_write(port, 0, UARTDM_DMEN);
1010 }
1011
1012 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1013 {
1014         unsigned int mr;
1015
1016         mr = msm_read(port, UART_MR1);
1017
1018         if (!(mctrl & TIOCM_RTS)) {
1019                 mr &= ~UART_MR1_RX_RDY_CTL;
1020                 msm_write(port, mr, UART_MR1);
1021                 msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1022         } else {
1023                 mr |= UART_MR1_RX_RDY_CTL;
1024                 msm_write(port, mr, UART_MR1);
1025         }
1026 }
1027
1028 static void msm_break_ctl(struct uart_port *port, int break_ctl)
1029 {
1030         if (break_ctl)
1031                 msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1032         else
1033                 msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1034 }
1035
1036 struct msm_baud_map {
1037         u16     divisor;
1038         u8      code;
1039         u8      rxstale;
1040 };
1041
1042 static const struct msm_baud_map *
1043 msm_find_best_baud(struct uart_port *port, unsigned int baud,
1044                    unsigned long *rate)
1045 {
1046         struct msm_port *msm_port = UART_TO_MSM(port);
1047         unsigned int divisor, result;
1048         unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1049         const struct msm_baud_map *entry, *end, *best;
1050         static const struct msm_baud_map table[] = {
1051                 {    1, 0xff, 31 },
1052                 {    2, 0xee, 16 },
1053                 {    3, 0xdd,  8 },
1054                 {    4, 0xcc,  6 },
1055                 {    6, 0xbb,  6 },
1056                 {    8, 0xaa,  6 },
1057                 {   12, 0x99,  6 },
1058                 {   16, 0x88,  1 },
1059                 {   24, 0x77,  1 },
1060                 {   32, 0x66,  1 },
1061                 {   48, 0x55,  1 },
1062                 {   96, 0x44,  1 },
1063                 {  192, 0x33,  1 },
1064                 {  384, 0x22,  1 },
1065                 {  768, 0x11,  1 },
1066                 { 1536, 0x00,  1 },
1067         };
1068
1069         best = table; /* Default to smallest divider */
1070         target = clk_round_rate(msm_port->clk, 16 * baud);
1071         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1072
1073         end = table + ARRAY_SIZE(table);
1074         entry = table;
1075         while (entry < end) {
1076                 if (entry->divisor <= divisor) {
1077                         result = target / entry->divisor / 16;
1078                         diff = abs(result - baud);
1079
1080                         /* Keep track of best entry */
1081                         if (diff < best_diff) {
1082                                 best_diff = diff;
1083                                 best = entry;
1084                                 best_rate = target;
1085                         }
1086
1087                         if (result == baud)
1088                                 break;
1089                 } else if (entry->divisor > divisor) {
1090                         old = target;
1091                         target = clk_round_rate(msm_port->clk, old + 1);
1092                         /*
1093                          * The rate didn't get any faster so we can't do
1094                          * better at dividing it down
1095                          */
1096                         if (target == old)
1097                                 break;
1098
1099                         /* Start the divisor search over at this new rate */
1100                         entry = table;
1101                         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1102                         continue;
1103                 }
1104                 entry++;
1105         }
1106
1107         *rate = best_rate;
1108         return best;
1109 }
1110
1111 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1112                              unsigned long *saved_flags)
1113 {
1114         unsigned int rxstale, watermark, mask;
1115         struct msm_port *msm_port = UART_TO_MSM(port);
1116         const struct msm_baud_map *entry;
1117         unsigned long flags, rate;
1118
1119         flags = *saved_flags;
1120         spin_unlock_irqrestore(&port->lock, flags);
1121
1122         entry = msm_find_best_baud(port, baud, &rate);
1123         clk_set_rate(msm_port->clk, rate);
1124         baud = rate / 16 / entry->divisor;
1125
1126         spin_lock_irqsave(&port->lock, flags);
1127         *saved_flags = flags;
1128         port->uartclk = rate;
1129
1130         msm_write(port, entry->code, UART_CSR);
1131
1132         /* RX stale watermark */
1133         rxstale = entry->rxstale;
1134         watermark = UART_IPR_STALE_LSB & rxstale;
1135         if (msm_port->is_uartdm) {
1136                 mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1137         } else {
1138                 watermark |= UART_IPR_RXSTALE_LAST;
1139                 mask = UART_IPR_STALE_TIMEOUT_MSB;
1140         }
1141
1142         watermark |= mask & (rxstale << 2);
1143
1144         msm_write(port, watermark, UART_IPR);
1145
1146         /* set RX watermark */
1147         watermark = (port->fifosize * 3) / 4;
1148         msm_write(port, watermark, UART_RFWR);
1149
1150         /* set TX watermark */
1151         msm_write(port, 10, UART_TFWR);
1152
1153         msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1154         msm_reset(port);
1155
1156         /* Enable RX and TX */
1157         msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1158
1159         /* turn on RX and CTS interrupts */
1160         msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1161                         UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1162
1163         msm_write(port, msm_port->imr, UART_IMR);
1164
1165         if (msm_port->is_uartdm) {
1166                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1167                 msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1168                 msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1169         }
1170
1171         return baud;
1172 }
1173
1174 static void msm_init_clock(struct uart_port *port)
1175 {
1176         struct msm_port *msm_port = UART_TO_MSM(port);
1177
1178         clk_prepare_enable(msm_port->clk);
1179         clk_prepare_enable(msm_port->pclk);
1180         msm_serial_set_mnd_regs(port);
1181 }
1182
1183 static int msm_startup(struct uart_port *port)
1184 {
1185         struct msm_port *msm_port = UART_TO_MSM(port);
1186         unsigned int data, rfr_level, mask;
1187         int ret;
1188
1189         snprintf(msm_port->name, sizeof(msm_port->name),
1190                  "msm_serial%d", port->line);
1191
1192         msm_init_clock(port);
1193
1194         if (likely(port->fifosize > 12))
1195                 rfr_level = port->fifosize - 12;
1196         else
1197                 rfr_level = port->fifosize;
1198
1199         /* set automatic RFR level */
1200         data = msm_read(port, UART_MR1);
1201
1202         if (msm_port->is_uartdm)
1203                 mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1204         else
1205                 mask = UART_MR1_AUTO_RFR_LEVEL1;
1206
1207         data &= ~mask;
1208         data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1209         data |= mask & (rfr_level << 2);
1210         data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1211         msm_write(port, data, UART_MR1);
1212
1213         if (msm_port->is_uartdm) {
1214                 msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1215                 msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1216         }
1217
1218         ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1219                           msm_port->name, port);
1220         if (unlikely(ret))
1221                 goto err_irq;
1222
1223         return 0;
1224
1225 err_irq:
1226         if (msm_port->is_uartdm)
1227                 msm_release_dma(msm_port);
1228
1229         clk_disable_unprepare(msm_port->pclk);
1230         clk_disable_unprepare(msm_port->clk);
1231
1232         return ret;
1233 }
1234
1235 static void msm_shutdown(struct uart_port *port)
1236 {
1237         struct msm_port *msm_port = UART_TO_MSM(port);
1238
1239         msm_port->imr = 0;
1240         msm_write(port, 0, UART_IMR); /* disable interrupts */
1241
1242         if (msm_port->is_uartdm)
1243                 msm_release_dma(msm_port);
1244
1245         clk_disable_unprepare(msm_port->clk);
1246
1247         free_irq(port->irq, port);
1248 }
1249
1250 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1251                             struct ktermios *old)
1252 {
1253         struct msm_port *msm_port = UART_TO_MSM(port);
1254         struct msm_dma *dma = &msm_port->rx_dma;
1255         unsigned long flags;
1256         unsigned int baud, mr;
1257
1258         spin_lock_irqsave(&port->lock, flags);
1259
1260         if (dma->chan) /* Terminate if any */
1261                 msm_stop_dma(port, dma);
1262
1263         /* calculate and set baud rate */
1264         baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1265         baud = msm_set_baud_rate(port, baud, &flags);
1266         if (tty_termios_baud_rate(termios))
1267                 tty_termios_encode_baud_rate(termios, baud, baud);
1268
1269         /* calculate parity */
1270         mr = msm_read(port, UART_MR2);
1271         mr &= ~UART_MR2_PARITY_MODE;
1272         if (termios->c_cflag & PARENB) {
1273                 if (termios->c_cflag & PARODD)
1274                         mr |= UART_MR2_PARITY_MODE_ODD;
1275                 else if (termios->c_cflag & CMSPAR)
1276                         mr |= UART_MR2_PARITY_MODE_SPACE;
1277                 else
1278                         mr |= UART_MR2_PARITY_MODE_EVEN;
1279         }
1280
1281         /* calculate bits per char */
1282         mr &= ~UART_MR2_BITS_PER_CHAR;
1283         switch (termios->c_cflag & CSIZE) {
1284         case CS5:
1285                 mr |= UART_MR2_BITS_PER_CHAR_5;
1286                 break;
1287         case CS6:
1288                 mr |= UART_MR2_BITS_PER_CHAR_6;
1289                 break;
1290         case CS7:
1291                 mr |= UART_MR2_BITS_PER_CHAR_7;
1292                 break;
1293         case CS8:
1294         default:
1295                 mr |= UART_MR2_BITS_PER_CHAR_8;
1296                 break;
1297         }
1298
1299         /* calculate stop bits */
1300         mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1301         if (termios->c_cflag & CSTOPB)
1302                 mr |= UART_MR2_STOP_BIT_LEN_TWO;
1303         else
1304                 mr |= UART_MR2_STOP_BIT_LEN_ONE;
1305
1306         /* set parity, bits per char, and stop bit */
1307         msm_write(port, mr, UART_MR2);
1308
1309         /* calculate and set hardware flow control */
1310         mr = msm_read(port, UART_MR1);
1311         mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1312         if (termios->c_cflag & CRTSCTS) {
1313                 mr |= UART_MR1_CTS_CTL;
1314                 mr |= UART_MR1_RX_RDY_CTL;
1315         }
1316         msm_write(port, mr, UART_MR1);
1317
1318         /* Configure status bits to ignore based on termio flags. */
1319         port->read_status_mask = 0;
1320         if (termios->c_iflag & INPCK)
1321                 port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1322         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1323                 port->read_status_mask |= UART_SR_RX_BREAK;
1324
1325         uart_update_timeout(port, termios->c_cflag, baud);
1326
1327         /* Try to use DMA */
1328         msm_start_rx_dma(msm_port);
1329
1330         spin_unlock_irqrestore(&port->lock, flags);
1331 }
1332
1333 static const char *msm_type(struct uart_port *port)
1334 {
1335         return "MSM";
1336 }
1337
1338 static void msm_release_port(struct uart_port *port)
1339 {
1340         struct platform_device *pdev = to_platform_device(port->dev);
1341         struct resource *uart_resource;
1342         resource_size_t size;
1343
1344         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1345         if (unlikely(!uart_resource))
1346                 return;
1347         size = resource_size(uart_resource);
1348
1349         release_mem_region(port->mapbase, size);
1350         iounmap(port->membase);
1351         port->membase = NULL;
1352 }
1353
1354 static int msm_request_port(struct uart_port *port)
1355 {
1356         struct platform_device *pdev = to_platform_device(port->dev);
1357         struct resource *uart_resource;
1358         resource_size_t size;
1359         int ret;
1360
1361         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1362         if (unlikely(!uart_resource))
1363                 return -ENXIO;
1364
1365         size = resource_size(uart_resource);
1366
1367         if (!request_mem_region(port->mapbase, size, "msm_serial"))
1368                 return -EBUSY;
1369
1370         port->membase = ioremap(port->mapbase, size);
1371         if (!port->membase) {
1372                 ret = -EBUSY;
1373                 goto fail_release_port;
1374         }
1375
1376         return 0;
1377
1378 fail_release_port:
1379         release_mem_region(port->mapbase, size);
1380         return ret;
1381 }
1382
1383 static void msm_config_port(struct uart_port *port, int flags)
1384 {
1385         int ret;
1386
1387         if (flags & UART_CONFIG_TYPE) {
1388                 port->type = PORT_MSM;
1389                 ret = msm_request_port(port);
1390                 if (ret)
1391                         return;
1392         }
1393 }
1394
1395 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1396 {
1397         if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1398                 return -EINVAL;
1399         if (unlikely(port->irq != ser->irq))
1400                 return -EINVAL;
1401         return 0;
1402 }
1403
1404 static void msm_power(struct uart_port *port, unsigned int state,
1405                       unsigned int oldstate)
1406 {
1407         struct msm_port *msm_port = UART_TO_MSM(port);
1408
1409         switch (state) {
1410         case 0:
1411                 clk_prepare_enable(msm_port->clk);
1412                 clk_prepare_enable(msm_port->pclk);
1413                 break;
1414         case 3:
1415                 clk_disable_unprepare(msm_port->clk);
1416                 clk_disable_unprepare(msm_port->pclk);
1417                 break;
1418         default:
1419                 pr_err("msm_serial: Unknown PM state %d\n", state);
1420         }
1421 }
1422
1423 #ifdef CONFIG_CONSOLE_POLL
1424 static int msm_poll_get_char_single(struct uart_port *port)
1425 {
1426         struct msm_port *msm_port = UART_TO_MSM(port);
1427         unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1428
1429         if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1430                 return NO_POLL_CHAR;
1431
1432         return msm_read(port, rf_reg) & 0xff;
1433 }
1434
1435 static int msm_poll_get_char_dm(struct uart_port *port)
1436 {
1437         int c;
1438         static u32 slop;
1439         static int count;
1440         unsigned char *sp = (unsigned char *)&slop;
1441
1442         /* Check if a previous read had more than one char */
1443         if (count) {
1444                 c = sp[sizeof(slop) - count];
1445                 count--;
1446         /* Or if FIFO is empty */
1447         } else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1448                 /*
1449                  * If RX packing buffer has less than a word, force stale to
1450                  * push contents into RX FIFO
1451                  */
1452                 count = msm_read(port, UARTDM_RXFS);
1453                 count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1454                 if (count) {
1455                         msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1456                         slop = msm_read(port, UARTDM_RF);
1457                         c = sp[0];
1458                         count--;
1459                         msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1460                         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1461                         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1462                                   UART_CR);
1463                 } else {
1464                         c = NO_POLL_CHAR;
1465                 }
1466         /* FIFO has a word */
1467         } else {
1468                 slop = msm_read(port, UARTDM_RF);
1469                 c = sp[0];
1470                 count = sizeof(slop) - 1;
1471         }
1472
1473         return c;
1474 }
1475
1476 static int msm_poll_get_char(struct uart_port *port)
1477 {
1478         u32 imr;
1479         int c;
1480         struct msm_port *msm_port = UART_TO_MSM(port);
1481
1482         /* Disable all interrupts */
1483         imr = msm_read(port, UART_IMR);
1484         msm_write(port, 0, UART_IMR);
1485
1486         if (msm_port->is_uartdm)
1487                 c = msm_poll_get_char_dm(port);
1488         else
1489                 c = msm_poll_get_char_single(port);
1490
1491         /* Enable interrupts */
1492         msm_write(port, imr, UART_IMR);
1493
1494         return c;
1495 }
1496
1497 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1498 {
1499         u32 imr;
1500         struct msm_port *msm_port = UART_TO_MSM(port);
1501
1502         /* Disable all interrupts */
1503         imr = msm_read(port, UART_IMR);
1504         msm_write(port, 0, UART_IMR);
1505
1506         if (msm_port->is_uartdm)
1507                 msm_reset_dm_count(port, 1);
1508
1509         /* Wait until FIFO is empty */
1510         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1511                 cpu_relax();
1512
1513         /* Write a character */
1514         msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1515
1516         /* Wait until FIFO is empty */
1517         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1518                 cpu_relax();
1519
1520         /* Enable interrupts */
1521         msm_write(port, imr, UART_IMR);
1522 }
1523 #endif
1524
1525 static struct uart_ops msm_uart_pops = {
1526         .tx_empty = msm_tx_empty,
1527         .set_mctrl = msm_set_mctrl,
1528         .get_mctrl = msm_get_mctrl,
1529         .stop_tx = msm_stop_tx,
1530         .start_tx = msm_start_tx,
1531         .stop_rx = msm_stop_rx,
1532         .enable_ms = msm_enable_ms,
1533         .break_ctl = msm_break_ctl,
1534         .startup = msm_startup,
1535         .shutdown = msm_shutdown,
1536         .set_termios = msm_set_termios,
1537         .type = msm_type,
1538         .release_port = msm_release_port,
1539         .request_port = msm_request_port,
1540         .config_port = msm_config_port,
1541         .verify_port = msm_verify_port,
1542         .pm = msm_power,
1543 #ifdef CONFIG_CONSOLE_POLL
1544         .poll_get_char  = msm_poll_get_char,
1545         .poll_put_char  = msm_poll_put_char,
1546 #endif
1547 };
1548
1549 static struct msm_port msm_uart_ports[] = {
1550         {
1551                 .uart = {
1552                         .iotype = UPIO_MEM,
1553                         .ops = &msm_uart_pops,
1554                         .flags = UPF_BOOT_AUTOCONF,
1555                         .fifosize = 64,
1556                         .line = 0,
1557                 },
1558         },
1559         {
1560                 .uart = {
1561                         .iotype = UPIO_MEM,
1562                         .ops = &msm_uart_pops,
1563                         .flags = UPF_BOOT_AUTOCONF,
1564                         .fifosize = 64,
1565                         .line = 1,
1566                 },
1567         },
1568         {
1569                 .uart = {
1570                         .iotype = UPIO_MEM,
1571                         .ops = &msm_uart_pops,
1572                         .flags = UPF_BOOT_AUTOCONF,
1573                         .fifosize = 64,
1574                         .line = 2,
1575                 },
1576         },
1577 };
1578
1579 #define UART_NR ARRAY_SIZE(msm_uart_ports)
1580
1581 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1582 {
1583         return &msm_uart_ports[line].uart;
1584 }
1585
1586 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1587 static void __msm_console_write(struct uart_port *port, const char *s,
1588                                 unsigned int count, bool is_uartdm)
1589 {
1590         unsigned long flags;
1591         int i;
1592         int num_newlines = 0;
1593         bool replaced = false;
1594         void __iomem *tf;
1595         int locked = 1;
1596
1597         if (is_uartdm)
1598                 tf = port->membase + UARTDM_TF;
1599         else
1600                 tf = port->membase + UART_TF;
1601
1602         /* Account for newlines that will get a carriage return added */
1603         for (i = 0; i < count; i++)
1604                 if (s[i] == '\n')
1605                         num_newlines++;
1606         count += num_newlines;
1607
1608         local_irq_save(flags);
1609
1610         if (port->sysrq)
1611                 locked = 0;
1612         else if (oops_in_progress)
1613                 locked = spin_trylock(&port->lock);
1614         else
1615                 spin_lock(&port->lock);
1616
1617         if (is_uartdm)
1618                 msm_reset_dm_count(port, count);
1619
1620         i = 0;
1621         while (i < count) {
1622                 int j;
1623                 unsigned int num_chars;
1624                 char buf[4] = { 0 };
1625
1626                 if (is_uartdm)
1627                         num_chars = min(count - i, (unsigned int)sizeof(buf));
1628                 else
1629                         num_chars = 1;
1630
1631                 for (j = 0; j < num_chars; j++) {
1632                         char c = *s;
1633
1634                         if (c == '\n' && !replaced) {
1635                                 buf[j] = '\r';
1636                                 j++;
1637                                 replaced = true;
1638                         }
1639                         if (j < num_chars) {
1640                                 buf[j] = c;
1641                                 s++;
1642                                 replaced = false;
1643                         }
1644                 }
1645
1646                 while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1647                         cpu_relax();
1648
1649                 iowrite32_rep(tf, buf, 1);
1650                 i += num_chars;
1651         }
1652
1653         if (locked)
1654                 spin_unlock(&port->lock);
1655
1656         local_irq_restore(flags);
1657 }
1658
1659 static void msm_console_write(struct console *co, const char *s,
1660                               unsigned int count)
1661 {
1662         struct uart_port *port;
1663         struct msm_port *msm_port;
1664
1665         BUG_ON(co->index < 0 || co->index >= UART_NR);
1666
1667         port = msm_get_port_from_line(co->index);
1668         msm_port = UART_TO_MSM(port);
1669
1670         __msm_console_write(port, s, count, msm_port->is_uartdm);
1671 }
1672
1673 static int __init msm_console_setup(struct console *co, char *options)
1674 {
1675         struct uart_port *port;
1676         int baud = 115200;
1677         int bits = 8;
1678         int parity = 'n';
1679         int flow = 'n';
1680
1681         if (unlikely(co->index >= UART_NR || co->index < 0))
1682                 return -ENXIO;
1683
1684         port = msm_get_port_from_line(co->index);
1685
1686         if (unlikely(!port->membase))
1687                 return -ENXIO;
1688
1689         msm_init_clock(port);
1690
1691         if (options)
1692                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1693
1694         pr_info("msm_serial: console setup on port #%d\n", port->line);
1695
1696         return uart_set_options(port, co, baud, parity, bits, flow);
1697 }
1698
1699 static void
1700 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1701 {
1702         struct earlycon_device *dev = con->data;
1703
1704         __msm_console_write(&dev->port, s, n, false);
1705 }
1706
1707 static int __init
1708 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1709 {
1710         if (!device->port.membase)
1711                 return -ENODEV;
1712
1713         device->con->write = msm_serial_early_write;
1714         return 0;
1715 }
1716 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1717                     msm_serial_early_console_setup);
1718
1719 static void
1720 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1721 {
1722         struct earlycon_device *dev = con->data;
1723
1724         __msm_console_write(&dev->port, s, n, true);
1725 }
1726
1727 static int __init
1728 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1729                                   const char *opt)
1730 {
1731         if (!device->port.membase)
1732                 return -ENODEV;
1733
1734         device->con->write = msm_serial_early_write_dm;
1735         return 0;
1736 }
1737 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1738                     msm_serial_early_console_setup_dm);
1739
1740 static struct uart_driver msm_uart_driver;
1741
1742 static struct console msm_console = {
1743         .name = "ttyMSM",
1744         .write = msm_console_write,
1745         .device = uart_console_device,
1746         .setup = msm_console_setup,
1747         .flags = CON_PRINTBUFFER,
1748         .index = -1,
1749         .data = &msm_uart_driver,
1750 };
1751
1752 #define MSM_CONSOLE     (&msm_console)
1753
1754 #else
1755 #define MSM_CONSOLE     NULL
1756 #endif
1757
1758 static struct uart_driver msm_uart_driver = {
1759         .owner = THIS_MODULE,
1760         .driver_name = "msm_serial",
1761         .dev_name = "ttyMSM",
1762         .nr = UART_NR,
1763         .cons = MSM_CONSOLE,
1764 };
1765
1766 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1767
1768 static const struct of_device_id msm_uartdm_table[] = {
1769         { .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1770         { .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1771         { .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1772         { .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1773         { }
1774 };
1775
1776 static int msm_serial_probe(struct platform_device *pdev)
1777 {
1778         struct msm_port *msm_port;
1779         struct resource *resource;
1780         struct uart_port *port;
1781         const struct of_device_id *id;
1782         int irq, line;
1783
1784         if (pdev->dev.of_node)
1785                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1786         else
1787                 line = pdev->id;
1788
1789         if (line < 0)
1790                 line = atomic_inc_return(&msm_uart_next_id) - 1;
1791
1792         if (unlikely(line < 0 || line >= UART_NR))
1793                 return -ENXIO;
1794
1795         dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1796
1797         port = msm_get_port_from_line(line);
1798         port->dev = &pdev->dev;
1799         msm_port = UART_TO_MSM(port);
1800
1801         id = of_match_device(msm_uartdm_table, &pdev->dev);
1802         if (id)
1803                 msm_port->is_uartdm = (unsigned long)id->data;
1804         else
1805                 msm_port->is_uartdm = 0;
1806
1807         msm_port->clk = devm_clk_get(&pdev->dev, "core");
1808         if (IS_ERR(msm_port->clk))
1809                 return PTR_ERR(msm_port->clk);
1810
1811         if (msm_port->is_uartdm) {
1812                 msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1813                 if (IS_ERR(msm_port->pclk))
1814                         return PTR_ERR(msm_port->pclk);
1815         }
1816
1817         port->uartclk = clk_get_rate(msm_port->clk);
1818         dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1819
1820         resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1821         if (unlikely(!resource))
1822                 return -ENXIO;
1823         port->mapbase = resource->start;
1824
1825         irq = platform_get_irq(pdev, 0);
1826         if (unlikely(irq < 0))
1827                 return -ENXIO;
1828         port->irq = irq;
1829
1830         platform_set_drvdata(pdev, port);
1831
1832         return uart_add_one_port(&msm_uart_driver, port);
1833 }
1834
1835 static int msm_serial_remove(struct platform_device *pdev)
1836 {
1837         struct uart_port *port = platform_get_drvdata(pdev);
1838
1839         uart_remove_one_port(&msm_uart_driver, port);
1840
1841         return 0;
1842 }
1843
1844 static const struct of_device_id msm_match_table[] = {
1845         { .compatible = "qcom,msm-uart" },
1846         { .compatible = "qcom,msm-uartdm" },
1847         {}
1848 };
1849 MODULE_DEVICE_TABLE(of, msm_match_table);
1850
1851 static struct platform_driver msm_platform_driver = {
1852         .remove = msm_serial_remove,
1853         .probe = msm_serial_probe,
1854         .driver = {
1855                 .name = "msm_serial",
1856                 .of_match_table = msm_match_table,
1857         },
1858 };
1859
1860 static int __init msm_serial_init(void)
1861 {
1862         int ret;
1863
1864         ret = uart_register_driver(&msm_uart_driver);
1865         if (unlikely(ret))
1866                 return ret;
1867
1868         ret = platform_driver_register(&msm_platform_driver);
1869         if (unlikely(ret))
1870                 uart_unregister_driver(&msm_uart_driver);
1871
1872         pr_info("msm_serial: driver initialized\n");
1873
1874         return ret;
1875 }
1876
1877 static void __exit msm_serial_exit(void)
1878 {
1879         platform_driver_unregister(&msm_platform_driver);
1880         uart_unregister_driver(&msm_uart_driver);
1881 }
1882
1883 module_init(msm_serial_init);
1884 module_exit(msm_serial_exit);
1885
1886 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1887 MODULE_DESCRIPTION("Driver for msm7x serial device");
1888 MODULE_LICENSE("GPL");