drivers/tty/serial/msm_serial.c

   1 /*
   2  * Driver for msm7k serial device and console
   3  *
   4  * Copyright (C) 2007 Google, Inc.
   5  * Author: Robert Love <rlove@google.com>
   6  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   7  *
   8  * This software is licensed under the terms of the GNU General Public
   9  * License version 2, as published by the Free Software Foundation, and
  10  * may be copied, distributed, and modified under those terms.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  */
  17
  18 #if defined(CONFIG_SERIAL_MSM_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
  19 # define SUPPORT_SYSRQ
  20 #endif
  21
  22 #include <linux/atomic.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/dmaengine.h>
  25 #include <linux/hrtimer.h>
  26 #include <linux/module.h>
  27 #include <linux/io.h>
  28 #include <linux/ioport.h>
  29 #include <linux/irq.h>
  30 #include <linux/init.h>
  31 #include <linux/console.h>
  32 #include <linux/tty.h>
  33 #include <linux/tty_flip.h>
  34 #include <linux/serial_core.h>
  35 #include <linux/serial.h>
  36 #include <linux/slab.h>
  37 #include <linux/clk.h>
  38 #include <linux/platform_device.h>
  39 #include <linux/delay.h>
  40 #include <linux/of.h>
  41 #include <linux/of_device.h>
  42
  43 #include "msm_serial.h"
  44
  45 #define UARTDM_BURST_SIZE       16   /* in bytes */
  46 #define UARTDM_TX_AIGN(x)       ((x) & ~0x3) /* valid for > 1p3 */
  47 #define UARTDM_TX_MAX           256   /* in bytes, valid for <= 1p3 */
  48 #define UARTDM_RX_SIZE          (UART_XMIT_SIZE / 4)
  49
  50 enum {
  51         UARTDM_1P1 = 1,
  52         UARTDM_1P2,
  53         UARTDM_1P3,
  54         UARTDM_1P4,
  55 };
  56
  57 struct msm_dma {
  58         struct dma_chan         *chan;
  59         enum dma_data_direction dir;
  60         dma_addr_t              phys;
  61         unsigned char           *virt;
  62         dma_cookie_t            cookie;
  63         u32                     enable_bit;
  64         unsigned int            count;
  65         struct dma_async_tx_descriptor  *desc;
  66 };
  67
  68 struct msm_port {
  69         struct uart_port        uart;
  70         char                    name[16];
  71         struct clk              *clk;
  72         struct clk              *pclk;
  73         unsigned int            imr;
  74         int                     is_uartdm;
  75         unsigned int            old_snap_state;
  76         bool                    break_detected;
  77         struct msm_dma          tx_dma;
  78         struct msm_dma          rx_dma;
  79 };
  80
  81 static void msm_handle_tx(struct uart_port *port);
  82 static void msm_start_rx_dma(struct msm_port *msm_port);
  83
  84 void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
  85 {
  86         struct device *dev = port->dev;
  87         unsigned int mapped;
  88         u32 val;
  89
  90         mapped = dma->count;
  91         dma->count = 0;
  92
  93         dmaengine_terminate_all(dma->chan);
  94
  95         /*
  96          * DMA Stall happens if enqueue and flush command happens concurrently.
  97          * For example before changing the baud rate/protocol configuration and
  98          * sending flush command to ADM, disable the channel of UARTDM.
  99          * Note: should not reset the receiver here immediately as it is not
 100          * suggested to do disable/reset or reset/disable at the same time.
 101          */
 102         val = msm_read(port, UARTDM_DMEN);
 103         val &= ~dma->enable_bit;
 104         msm_write(port, val, UARTDM_DMEN);
 105
 106         if (mapped)
 107                 dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 108 }
 109
 110 static void msm_release_dma(struct msm_port *msm_port)
 111 {
 112         struct msm_dma *dma;
 113
 114         dma = &msm_port->tx_dma;
 115         if (dma->chan) {
 116                 msm_stop_dma(&msm_port->uart, dma);
 117                 dma_release_channel(dma->chan);
 118         }
 119
 120         memset(dma, 0, sizeof(*dma));
 121
 122         dma = &msm_port->rx_dma;
 123         if (dma->chan) {
 124                 msm_stop_dma(&msm_port->uart, dma);
 125                 dma_release_channel(dma->chan);
 126                 kfree(dma->virt);
 127         }
 128
 129         memset(dma, 0, sizeof(*dma));
 130 }
 131
 132 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 133 {
 134         struct device *dev = msm_port->uart.dev;
 135         struct dma_slave_config conf;
 136         struct msm_dma *dma;
 137         u32 crci = 0;
 138         int ret;
 139
 140         dma = &msm_port->tx_dma;
 141
 142         /* allocate DMA resources, if available */
 143         dma->chan = dma_request_slave_channel_reason(dev, "tx");
 144         if (IS_ERR(dma->chan))
 145                 goto no_tx;
 146
 147         of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 148
 149         memset(&conf, 0, sizeof(conf));
 150         conf.direction = DMA_MEM_TO_DEV;
 151         conf.device_fc = true;
 152         conf.dst_addr = base + UARTDM_TF;
 153         conf.dst_maxburst = UARTDM_BURST_SIZE;
 154         conf.slave_id = crci;
 155
 156         ret = dmaengine_slave_config(dma->chan, &conf);
 157         if (ret)
 158                 goto rel_tx;
 159
 160         dma->dir = DMA_TO_DEVICE;
 161
 162         if (msm_port->is_uartdm < UARTDM_1P4)
 163                 dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 164         else
 165                 dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 166
 167         return;
 168
 169 rel_tx:
 170         dma_release_channel(dma->chan);
 171 no_tx:
 172         memset(dma, 0, sizeof(*dma));
 173 }
 174
 175 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 176 {
 177         struct device *dev = msm_port->uart.dev;
 178         struct dma_slave_config conf;
 179         struct msm_dma *dma;
 180         u32 crci = 0;
 181         int ret;
 182
 183         dma = &msm_port->rx_dma;
 184
 185         /* allocate DMA resources, if available */
 186         dma->chan = dma_request_slave_channel_reason(dev, "rx");
 187         if (IS_ERR(dma->chan))
 188                 goto no_rx;
 189
 190         of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 191
 192         dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 193         if (!dma->virt)
 194                 goto rel_rx;
 195
 196         memset(&conf, 0, sizeof(conf));
 197         conf.direction = DMA_DEV_TO_MEM;
 198         conf.device_fc = true;
 199         conf.src_addr = base + UARTDM_RF;
 200         conf.src_maxburst = UARTDM_BURST_SIZE;
 201         conf.slave_id = crci;
 202
 203         ret = dmaengine_slave_config(dma->chan, &conf);
 204         if (ret)
 205                 goto err;
 206
 207         dma->dir = DMA_FROM_DEVICE;
 208
 209         if (msm_port->is_uartdm < UARTDM_1P4)
 210                 dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 211         else
 212                 dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 213
 214         return;
 215 err:
 216         kfree(dma->virt);
 217 rel_rx:
 218         dma_release_channel(dma->chan);
 219 no_rx:
 220         memset(dma, 0, sizeof(*dma));
 221 }
 222
 223 static inline void msm_wait_for_xmitr(struct uart_port *port)
 224 {
 225         unsigned int timeout = 500000;
 226
 227         while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 228                 if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 229                         break;
 230                 udelay(1);
 231                 if (!timeout--)
 232                         break;
 233         }
 234         msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 235 }
 236
 237 static void msm_stop_tx(struct uart_port *port)
 238 {
 239         struct msm_port *msm_port = UART_TO_MSM(port);
 240
 241         msm_port->imr &= ~UART_IMR_TXLEV;
 242         msm_write(port, msm_port->imr, UART_IMR);
 243 }
 244
 245 static void msm_start_tx(struct uart_port *port)
 246 {
 247         struct msm_port *msm_port = UART_TO_MSM(port);
 248         struct msm_dma *dma = &msm_port->tx_dma;
 249
 250         /* Already started in DMA mode */
 251         if (dma->count)
 252                 return;
 253
 254         msm_port->imr |= UART_IMR_TXLEV;
 255         msm_write(port, msm_port->imr, UART_IMR);
 256 }
 257
 258 static void msm_reset_dm_count(struct uart_port *port, int count)
 259 {
 260         msm_wait_for_xmitr(port);
 261         msm_write(port, count, UARTDM_NCF_TX);
 262         msm_read(port, UARTDM_NCF_TX);
 263 }
 264
 265 static void msm_complete_tx_dma(void *args)
 266 {
 267         struct msm_port *msm_port = args;
 268         struct uart_port *port = &msm_port->uart;
 269         struct circ_buf *xmit = &port->state->xmit;
 270         struct msm_dma *dma = &msm_port->tx_dma;
 271         struct dma_tx_state state;
 272         enum dma_status status;
 273         unsigned long flags;
 274         unsigned int count;
 275         u32 val;
 276
 277         spin_lock_irqsave(&port->lock, flags);
 278
 279         /* Already stopped */
 280         if (!dma->count)
 281                 goto done;
 282
 283         status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
 284
 285         dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 286
 287         val = msm_read(port, UARTDM_DMEN);
 288         val &= ~dma->enable_bit;
 289         msm_write(port, val, UARTDM_DMEN);
 290
 291         if (msm_port->is_uartdm > UARTDM_1P3) {
 292                 msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 293                 msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 294         }
 295
 296         count = dma->count - state.residue;
 297         port->icount.tx += count;
 298         dma->count = 0;
 299
 300         xmit->tail += count;
 301         xmit->tail &= UART_XMIT_SIZE - 1;
 302
 303         /* Restore "Tx FIFO below watermark" interrupt */
 304         msm_port->imr |= UART_IMR_TXLEV;
 305         msm_write(port, msm_port->imr, UART_IMR);
 306
 307         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 308                 uart_write_wakeup(port);
 309
 310         msm_handle_tx(port);
 311 done:
 312         spin_unlock_irqrestore(&port->lock, flags);
 313 }
 314
 315 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 316 {
 317         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 318         struct uart_port *port = &msm_port->uart;
 319         struct msm_dma *dma = &msm_port->tx_dma;
 320         void *cpu_addr;
 321         int ret;
 322         u32 val;
 323
 324         cpu_addr = &xmit->buf[xmit->tail];
 325
 326         dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 327         ret = dma_mapping_error(port->dev, dma->phys);
 328         if (ret)
 329                 return ret;
 330
 331         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 332                                                 count, DMA_MEM_TO_DEV,
 333                                                 DMA_PREP_INTERRUPT |
 334                                                 DMA_PREP_FENCE);
 335         if (!dma->desc) {
 336                 ret = -EIO;
 337                 goto unmap;
 338         }
 339
 340         dma->desc->callback = msm_complete_tx_dma;
 341         dma->desc->callback_param = msm_port;
 342
 343         dma->cookie = dmaengine_submit(dma->desc);
 344         ret = dma_submit_error(dma->cookie);
 345         if (ret)
 346                 goto unmap;
 347
 348         /*
 349          * Using DMA complete for Tx FIFO reload, no need for
 350          * "Tx FIFO below watermark" one, disable it
 351          */
 352         msm_port->imr &= ~UART_IMR_TXLEV;
 353         msm_write(port, msm_port->imr, UART_IMR);
 354
 355         dma->count = count;
 356
 357         val = msm_read(port, UARTDM_DMEN);
 358         val |= dma->enable_bit;
 359
 360         if (msm_port->is_uartdm < UARTDM_1P4)
 361                 msm_write(port, val, UARTDM_DMEN);
 362
 363         msm_reset_dm_count(port, count);
 364
 365         if (msm_port->is_uartdm > UARTDM_1P3)
 366                 msm_write(port, val, UARTDM_DMEN);
 367
 368         dma_async_issue_pending(dma->chan);
 369         return 0;
 370 unmap:
 371         dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 372         return ret;
 373 }
 374
 375 static void msm_complete_rx_dma(void *args)
 376 {
 377         struct msm_port *msm_port = args;
 378         struct uart_port *port = &msm_port->uart;
 379         struct tty_port *tport = &port->state->port;
 380         struct msm_dma *dma = &msm_port->rx_dma;
 381         int count = 0, i, sysrq;
 382         unsigned long flags;
 383         u32 val;
 384
 385         spin_lock_irqsave(&port->lock, flags);
 386
 387         /* Already stopped */
 388         if (!dma->count)
 389                 goto done;
 390
 391         val = msm_read(port, UARTDM_DMEN);
 392         val &= ~dma->enable_bit;
 393         msm_write(port, val, UARTDM_DMEN);
 394
 395         /* Restore interrupts */
 396         msm_port->imr |= UART_IMR_RXLEV | UART_IMR_RXSTALE;
 397         msm_write(port, msm_port->imr, UART_IMR);
 398
 399         if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 400                 port->icount.overrun++;
 401                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 402                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 403         }
 404
 405         count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 406
 407         port->icount.rx += count;
 408
 409         dma->count = 0;
 410
 411         dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 412
 413         for (i = 0; i < count; i++) {
 414                 char flag = TTY_NORMAL;
 415
 416                 if (msm_port->break_detected && dma->virt[i] == 0) {
 417                         port->icount.brk++;
 418                         flag = TTY_BREAK;
 419                         msm_port->break_detected = false;
 420                         if (uart_handle_break(port))
 421                                 continue;
 422                 }
 423
 424                 if (!(port->read_status_mask & UART_SR_RX_BREAK))
 425                         flag = TTY_NORMAL;
 426
 427                 spin_unlock_irqrestore(&port->lock, flags);
 428                 sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 429                 spin_lock_irqsave(&port->lock, flags);
 430                 if (!sysrq)
 431                         tty_insert_flip_char(tport, dma->virt[i], flag);
 432         }
 433
 434         msm_start_rx_dma(msm_port);
 435 done:
 436         spin_unlock_irqrestore(&port->lock, flags);
 437
 438         if (count)
 439                 tty_flip_buffer_push(tport);
 440 }
 441
 442 static void msm_start_rx_dma(struct msm_port *msm_port)
 443 {
 444         struct msm_dma *dma = &msm_port->rx_dma;
 445         struct uart_port *uart = &msm_port->uart;
 446         u32 val;
 447         int ret;
 448
 449         if (!dma->chan)
 450                 return;
 451
 452         dma->phys = dma_map_single(uart->dev, dma->virt,
 453                                    UARTDM_RX_SIZE, dma->dir);
 454         ret = dma_mapping_error(uart->dev, dma->phys);
 455         if (ret)
 456                 return;
 457
 458         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 459                                                 UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 460                                                 DMA_PREP_INTERRUPT);
 461         if (!dma->desc)
 462                 goto unmap;
 463
 464         dma->desc->callback = msm_complete_rx_dma;
 465         dma->desc->callback_param = msm_port;
 466
 467         dma->cookie = dmaengine_submit(dma->desc);
 468         ret = dma_submit_error(dma->cookie);
 469         if (ret)
 470                 goto unmap;
 471         /*
 472          * Using DMA for FIFO off-load, no need for "Rx FIFO over
 473          * watermark" or "stale" interrupts, disable them
 474          */
 475         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 476
 477         /*
 478          * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 479          * we need RXSTALE to flush input DMA fifo to memory
 480          */
 481         if (msm_port->is_uartdm < UARTDM_1P4)
 482                 msm_port->imr |= UART_IMR_RXSTALE;
 483
 484         msm_write(uart, msm_port->imr, UART_IMR);
 485
 486         dma->count = UARTDM_RX_SIZE;
 487
 488         dma_async_issue_pending(dma->chan);
 489
 490         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 491         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 492
 493         val = msm_read(uart, UARTDM_DMEN);
 494         val |= dma->enable_bit;
 495
 496         if (msm_port->is_uartdm < UARTDM_1P4)
 497                 msm_write(uart, val, UARTDM_DMEN);
 498
 499         msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 500
 501         if (msm_port->is_uartdm > UARTDM_1P3)
 502                 msm_write(uart, val, UARTDM_DMEN);
 503
 504         return;
 505 unmap:
 506         dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 507 }
 508
 509 static void msm_stop_rx(struct uart_port *port)
 510 {
 511         struct msm_port *msm_port = UART_TO_MSM(port);
 512         struct msm_dma *dma = &msm_port->rx_dma;
 513
 514         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 515         msm_write(port, msm_port->imr, UART_IMR);
 516
 517         if (dma->chan)
 518                 msm_stop_dma(port, dma);
 519 }
 520
 521 static void msm_enable_ms(struct uart_port *port)
 522 {
 523         struct msm_port *msm_port = UART_TO_MSM(port);
 524
 525         msm_port->imr |= UART_IMR_DELTA_CTS;
 526         msm_write(port, msm_port->imr, UART_IMR);
 527 }
 528
 529 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 530 {
 531         struct tty_port *tport = &port->state->port;
 532         unsigned int sr;
 533         int count = 0;
 534         struct msm_port *msm_port = UART_TO_MSM(port);
 535
 536         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 537                 port->icount.overrun++;
 538                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 539                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 540         }
 541
 542         if (misr & UART_IMR_RXSTALE) {
 543                 count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 544                         msm_port->old_snap_state;
 545                 msm_port->old_snap_state = 0;
 546         } else {
 547                 count = 4 * (msm_read(port, UART_RFWR));
 548                 msm_port->old_snap_state += count;
 549         }
 550
 551         /* TODO: Precise error reporting */
 552
 553         port->icount.rx += count;
 554
 555         while (count > 0) {
 556                 unsigned char buf[4];
 557                 int sysrq, r_count, i;
 558
 559                 sr = msm_read(port, UART_SR);
 560                 if ((sr & UART_SR_RX_READY) == 0) {
 561                         msm_port->old_snap_state -= count;
 562                         break;
 563                 }
 564
 565                 ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 566                 r_count = min_t(int, count, sizeof(buf));
 567
 568                 for (i = 0; i < r_count; i++) {
 569                         char flag = TTY_NORMAL;
 570
 571                         if (msm_port->break_detected && buf[i] == 0) {
 572                                 port->icount.brk++;
 573                                 flag = TTY_BREAK;
 574                                 msm_port->break_detected = false;
 575                                 if (uart_handle_break(port))
 576                                         continue;
 577                         }
 578
 579                         if (!(port->read_status_mask & UART_SR_RX_BREAK))
 580                                 flag = TTY_NORMAL;
 581
 582                         spin_unlock(&port->lock);
 583                         sysrq = uart_handle_sysrq_char(port, buf[i]);
 584                         spin_lock(&port->lock);
 585                         if (!sysrq)
 586                                 tty_insert_flip_char(tport, buf[i], flag);
 587                 }
 588                 count -= r_count;
 589         }
 590
 591         spin_unlock(&port->lock);
 592         tty_flip_buffer_push(tport);
 593         spin_lock(&port->lock);
 594
 595         if (misr & (UART_IMR_RXSTALE))
 596                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 597         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 598         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 599
 600         /* Try to use DMA */
 601         msm_start_rx_dma(msm_port);
 602 }
 603
 604 static void msm_handle_rx(struct uart_port *port)
 605 {
 606         struct tty_port *tport = &port->state->port;
 607         unsigned int sr;
 608
 609         /*
 610          * Handle overrun. My understanding of the hardware is that overrun
 611          * is not tied to the RX buffer, so we handle the case out of band.
 612          */
 613         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 614                 port->icount.overrun++;
 615                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 616                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 617         }
 618
 619         /* and now the main RX loop */
 620         while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 621                 unsigned int c;
 622                 char flag = TTY_NORMAL;
 623                 int sysrq;
 624
 625                 c = msm_read(port, UART_RF);
 626
 627                 if (sr & UART_SR_RX_BREAK) {
 628                         port->icount.brk++;
 629                         if (uart_handle_break(port))
 630                                 continue;
 631                 } else if (sr & UART_SR_PAR_FRAME_ERR) {
 632                         port->icount.frame++;
 633                 } else {
 634                         port->icount.rx++;
 635                 }
 636
 637                 /* Mask conditions we're ignorning. */
 638                 sr &= port->read_status_mask;
 639
 640                 if (sr & UART_SR_RX_BREAK)
 641                         flag = TTY_BREAK;
 642                 else if (sr & UART_SR_PAR_FRAME_ERR)
 643                         flag = TTY_FRAME;
 644
 645                 spin_unlock(&port->lock);
 646                 sysrq = uart_handle_sysrq_char(port, c);
 647                 spin_lock(&port->lock);
 648                 if (!sysrq)
 649                         tty_insert_flip_char(tport, c, flag);
 650         }
 651
 652         spin_unlock(&port->lock);
 653         tty_flip_buffer_push(tport);
 654         spin_lock(&port->lock);
 655 }
 656
 657 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 658 {
 659         struct circ_buf *xmit = &port->state->xmit;
 660         struct msm_port *msm_port = UART_TO_MSM(port);
 661         unsigned int num_chars;
 662         unsigned int tf_pointer = 0;
 663         void __iomem *tf;
 664
 665         if (msm_port->is_uartdm)
 666                 tf = port->membase + UARTDM_TF;
 667         else
 668                 tf = port->membase + UART_TF;
 669
 670         if (tx_count && msm_port->is_uartdm)
 671                 msm_reset_dm_count(port, tx_count);
 672
 673         while (tf_pointer < tx_count) {
 674                 int i;
 675                 char buf[4] = { 0 };
 676
 677                 if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 678                         break;
 679
 680                 if (msm_port->is_uartdm)
 681                         num_chars = min(tx_count - tf_pointer,
 682                                         (unsigned int)sizeof(buf));
 683                 else
 684                         num_chars = 1;
 685
 686                 for (i = 0; i < num_chars; i++) {
 687                         buf[i] = xmit->buf[xmit->tail + i];
 688                         port->icount.tx++;
 689                 }
 690
 691                 iowrite32_rep(tf, buf, 1);
 692                 xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 693                 tf_pointer += num_chars;
 694         }
 695
 696         /* disable tx interrupts if nothing more to send */
 697         if (uart_circ_empty(xmit))
 698                 msm_stop_tx(port);
 699
 700         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 701                 uart_write_wakeup(port);
 702 }
 703
 704 static void msm_handle_tx(struct uart_port *port)
 705 {
 706         struct msm_port *msm_port = UART_TO_MSM(port);
 707         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 708         struct msm_dma *dma = &msm_port->tx_dma;
 709         unsigned int pio_count, dma_count, dma_min;
 710         char buf[4] = { 0 };
 711         void __iomem *tf;
 712         int err = 0;
 713
 714         if (port->x_char) {
 715                 if (msm_port->is_uartdm)
 716                         tf = port->membase + UARTDM_TF;
 717                 else
 718                         tf = port->membase + UART_TF;
 719
 720                 buf[0] = port->x_char;
 721
 722                 if (msm_port->is_uartdm)
 723                         msm_reset_dm_count(port, 1);
 724
 725                 iowrite32_rep(tf, buf, 1);
 726                 port->icount.tx++;
 727                 port->x_char = 0;
 728                 return;
 729         }
 730
 731         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 732                 msm_stop_tx(port);
 733                 return;
 734         }
 735
 736         pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 737         dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 738
 739         dma_min = 1;    /* Always DMA */
 740         if (msm_port->is_uartdm > UARTDM_1P3) {
 741                 dma_count = UARTDM_TX_AIGN(dma_count);
 742                 dma_min = UARTDM_BURST_SIZE;
 743         } else {
 744                 if (dma_count > UARTDM_TX_MAX)
 745                         dma_count = UARTDM_TX_MAX;
 746         }
 747
 748         if (pio_count > port->fifosize)
 749                 pio_count = port->fifosize;
 750
 751         if (!dma->chan || dma_count < dma_min)
 752                 msm_handle_tx_pio(port, pio_count);
 753         else
 754                 err = msm_handle_tx_dma(msm_port, dma_count);
 755
 756         if (err)        /* fall back to PIO mode */
 757                 msm_handle_tx_pio(port, pio_count);
 758 }
 759
 760 static void msm_handle_delta_cts(struct uart_port *port)
 761 {
 762         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 763         port->icount.cts++;
 764         wake_up_interruptible(&port->state->port.delta_msr_wait);
 765 }
 766
 767 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 768 {
 769         struct uart_port *port = dev_id;
 770         struct msm_port *msm_port = UART_TO_MSM(port);
 771         struct msm_dma *dma = &msm_port->rx_dma;
 772         unsigned long flags;
 773         unsigned int misr;
 774         u32 val;
 775
 776         spin_lock_irqsave(&port->lock, flags);
 777         misr = msm_read(port, UART_MISR);
 778         msm_write(port, 0, UART_IMR); /* disable interrupt */
 779
 780         if (misr & UART_IMR_RXBREAK_START) {
 781                 msm_port->break_detected = true;
 782                 msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 783         }
 784
 785         if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 786                 if (dma->count) {
 787                         val = UART_CR_CMD_STALE_EVENT_DISABLE;
 788                         msm_write(port, val, UART_CR);
 789                         val = UART_CR_CMD_RESET_STALE_INT;
 790                         msm_write(port, val, UART_CR);
 791                         /*
 792                          * Flush DMA input fifo to memory, this will also
 793                          * trigger DMA RX completion
 794                          */
 795                         dmaengine_terminate_all(dma->chan);
 796                 } else if (msm_port->is_uartdm) {
 797                         msm_handle_rx_dm(port, misr);
 798                 } else {
 799                         msm_handle_rx(port);
 800                 }
 801         }
 802         if (misr & UART_IMR_TXLEV)
 803                 msm_handle_tx(port);
 804         if (misr & UART_IMR_DELTA_CTS)
 805                 msm_handle_delta_cts(port);
 806
 807         msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 808         spin_unlock_irqrestore(&port->lock, flags);
 809
 810         return IRQ_HANDLED;
 811 }
 812
 813 static unsigned int msm_tx_empty(struct uart_port *port)
 814 {
 815         return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 816 }
 817
 818 static unsigned int msm_get_mctrl(struct uart_port *port)
 819 {
 820         return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
 821 }
 822
 823 static void msm_reset(struct uart_port *port)
 824 {
 825         struct msm_port *msm_port = UART_TO_MSM(port);
 826         unsigned int mr;
 827
 828         /* reset everything */
 829         msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
 830         msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 831         msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 832         msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
 833         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 834         msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
 835         mr = msm_read(port, UART_MR1);
 836         mr &= ~UART_MR1_RX_RDY_CTL;
 837         msm_write(port, mr, UART_MR1);
 838
 839         /* Disable DM modes */
 840         if (msm_port->is_uartdm)
 841                 msm_write(port, 0, UARTDM_DMEN);
 842 }
 843
 844 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
 845 {
 846         unsigned int mr;
 847
 848         mr = msm_read(port, UART_MR1);
 849
 850         if (!(mctrl & TIOCM_RTS)) {
 851                 mr &= ~UART_MR1_RX_RDY_CTL;
 852                 msm_write(port, mr, UART_MR1);
 853                 msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
 854         } else {
 855                 mr |= UART_MR1_RX_RDY_CTL;
 856                 msm_write(port, mr, UART_MR1);
 857         }
 858 }
 859
 860 static void msm_break_ctl(struct uart_port *port, int break_ctl)
 861 {
 862         if (break_ctl)
 863                 msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
 864         else
 865                 msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
 866 }
 867
 868 struct msm_baud_map {
 869         u16     divisor;
 870         u8      code;
 871         u8      rxstale;
 872 };
 873
 874 static const struct msm_baud_map *
 875 msm_find_best_baud(struct uart_port *port, unsigned int baud,
 876                    unsigned long *rate)
 877 {
 878         struct msm_port *msm_port = UART_TO_MSM(port);
 879         unsigned int divisor, result;
 880         unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
 881         const struct msm_baud_map *entry, *end, *best;
 882         static const struct msm_baud_map table[] = {
 883                 {    1, 0xff, 31 },
 884                 {    2, 0xee, 16 },
 885                 {    3, 0xdd,  8 },
 886                 {    4, 0xcc,  6 },
 887                 {    6, 0xbb,  6 },
 888                 {    8, 0xaa,  6 },
 889                 {   12, 0x99,  6 },
 890                 {   16, 0x88,  1 },
 891                 {   24, 0x77,  1 },
 892                 {   32, 0x66,  1 },
 893                 {   48, 0x55,  1 },
 894                 {   96, 0x44,  1 },
 895                 {  192, 0x33,  1 },
 896                 {  384, 0x22,  1 },
 897                 {  768, 0x11,  1 },
 898                 { 1536, 0x00,  1 },
 899         };
 900
 901         best = table; /* Default to smallest divider */
 902         target = clk_round_rate(msm_port->clk, 16 * baud);
 903         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
 904
 905         end = table + ARRAY_SIZE(table);
 906         entry = table;
 907         while (entry < end) {
 908                 if (entry->divisor <= divisor) {
 909                         result = target / entry->divisor / 16;
 910                         diff = abs(result - baud);
 911
 912                         /* Keep track of best entry */
 913                         if (diff < best_diff) {
 914                                 best_diff = diff;
 915                                 best = entry;
 916                                 best_rate = target;
 917                         }
 918
 919                         if (result == baud)
 920                                 break;
 921                 } else if (entry->divisor > divisor) {
 922                         old = target;
 923                         target = clk_round_rate(msm_port->clk, old + 1);
 924                         /*
 925                          * The rate didn't get any faster so we can't do
 926                          * better at dividing it down
 927                          */
 928                         if (target == old)
 929                                 break;
 930
 931                         /* Start the divisor search over at this new rate */
 932                         entry = table;
 933                         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
 934                         continue;
 935                 }
 936                 entry++;
 937         }
 938
 939         *rate = best_rate;
 940         return best;
 941 }
 942
 943 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
 944                              unsigned long *saved_flags)
 945 {
 946         unsigned int rxstale, watermark, mask;
 947         struct msm_port *msm_port = UART_TO_MSM(port);
 948         const struct msm_baud_map *entry;
 949         unsigned long flags, rate;
 950
 951         flags = *saved_flags;
 952         spin_unlock_irqrestore(&port->lock, flags);
 953
 954         entry = msm_find_best_baud(port, baud, &rate);
 955         clk_set_rate(msm_port->clk, rate);
 956         baud = rate / 16 / entry->divisor;
 957
 958         spin_lock_irqsave(&port->lock, flags);
 959         *saved_flags = flags;
 960         port->uartclk = rate;
 961
 962         msm_write(port, entry->code, UART_CSR);
 963
 964         /* RX stale watermark */
 965         rxstale = entry->rxstale;
 966         watermark = UART_IPR_STALE_LSB & rxstale;
 967         if (msm_port->is_uartdm) {
 968                 mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
 969         } else {
 970                 watermark |= UART_IPR_RXSTALE_LAST;
 971                 mask = UART_IPR_STALE_TIMEOUT_MSB;
 972         }
 973
 974         watermark |= mask & (rxstale << 2);
 975
 976         msm_write(port, watermark, UART_IPR);
 977
 978         /* set RX watermark */
 979         watermark = (port->fifosize * 3) / 4;
 980         msm_write(port, watermark, UART_RFWR);
 981
 982         /* set TX watermark */
 983         msm_write(port, 10, UART_TFWR);
 984
 985         msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
 986         msm_reset(port);
 987
 988         /* Enable RX and TX */
 989         msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
 990
 991         /* turn on RX and CTS interrupts */
 992         msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
 993                         UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
 994
 995         msm_write(port, msm_port->imr, UART_IMR);
 996
 997         if (msm_port->is_uartdm) {
 998                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 999                 msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1000                 msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1001         }
1002
1003         return baud;
1004 }
1005
1006 static void msm_init_clock(struct uart_port *port)
1007 {
1008         struct msm_port *msm_port = UART_TO_MSM(port);
1009
1010         clk_prepare_enable(msm_port->clk);
1011         clk_prepare_enable(msm_port->pclk);
1012         msm_serial_set_mnd_regs(port);
1013 }
1014
1015 static int msm_startup(struct uart_port *port)
1016 {
1017         struct msm_port *msm_port = UART_TO_MSM(port);
1018         unsigned int data, rfr_level, mask;
1019         int ret;
1020
1021         snprintf(msm_port->name, sizeof(msm_port->name),
1022                  "msm_serial%d", port->line);
1023
1024         ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1025                           msm_port->name, port);
1026         if (unlikely(ret))
1027                 return ret;
1028
1029         msm_init_clock(port);
1030
1031         if (likely(port->fifosize > 12))
1032                 rfr_level = port->fifosize - 12;
1033         else
1034                 rfr_level = port->fifosize;
1035
1036         /* set automatic RFR level */
1037         data = msm_read(port, UART_MR1);
1038
1039         if (msm_port->is_uartdm)
1040                 mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1041         else
1042                 mask = UART_MR1_AUTO_RFR_LEVEL1;
1043
1044         data &= ~mask;
1045         data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1046         data |= mask & (rfr_level << 2);
1047         data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1048         msm_write(port, data, UART_MR1);
1049
1050         if (msm_port->is_uartdm) {
1051                 msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1052                 msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1053         }
1054
1055         return 0;
1056 }
1057
1058 static void msm_shutdown(struct uart_port *port)
1059 {
1060         struct msm_port *msm_port = UART_TO_MSM(port);
1061
1062         msm_port->imr = 0;
1063         msm_write(port, 0, UART_IMR); /* disable interrupts */
1064
1065         if (msm_port->is_uartdm)
1066                 msm_release_dma(msm_port);
1067
1068         clk_disable_unprepare(msm_port->clk);
1069
1070         free_irq(port->irq, port);
1071 }
1072
1073 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1074                             struct ktermios *old)
1075 {
1076         struct msm_port *msm_port = UART_TO_MSM(port);
1077         struct msm_dma *dma = &msm_port->rx_dma;
1078         unsigned long flags;
1079         unsigned int baud, mr;
1080
1081         spin_lock_irqsave(&port->lock, flags);
1082
1083         if (dma->chan) /* Terminate if any */
1084                 msm_stop_dma(port, dma);
1085
1086         /* calculate and set baud rate */
1087         baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1088         baud = msm_set_baud_rate(port, baud, &flags);
1089         if (tty_termios_baud_rate(termios))
1090                 tty_termios_encode_baud_rate(termios, baud, baud);
1091
1092         /* calculate parity */
1093         mr = msm_read(port, UART_MR2);
1094         mr &= ~UART_MR2_PARITY_MODE;
1095         if (termios->c_cflag & PARENB) {
1096                 if (termios->c_cflag & PARODD)
1097                         mr |= UART_MR2_PARITY_MODE_ODD;
1098                 else if (termios->c_cflag & CMSPAR)
1099                         mr |= UART_MR2_PARITY_MODE_SPACE;
1100                 else
1101                         mr |= UART_MR2_PARITY_MODE_EVEN;
1102         }
1103
1104         /* calculate bits per char */
1105         mr &= ~UART_MR2_BITS_PER_CHAR;
1106         switch (termios->c_cflag & CSIZE) {
1107         case CS5:
1108                 mr |= UART_MR2_BITS_PER_CHAR_5;
1109                 break;
1110         case CS6:
1111                 mr |= UART_MR2_BITS_PER_CHAR_6;
1112                 break;
1113         case CS7:
1114                 mr |= UART_MR2_BITS_PER_CHAR_7;
1115                 break;
1116         case CS8:
1117         default:
1118                 mr |= UART_MR2_BITS_PER_CHAR_8;
1119                 break;
1120         }
1121
1122         /* calculate stop bits */
1123         mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1124         if (termios->c_cflag & CSTOPB)
1125                 mr |= UART_MR2_STOP_BIT_LEN_TWO;
1126         else
1127                 mr |= UART_MR2_STOP_BIT_LEN_ONE;
1128
1129         /* set parity, bits per char, and stop bit */
1130         msm_write(port, mr, UART_MR2);
1131
1132         /* calculate and set hardware flow control */
1133         mr = msm_read(port, UART_MR1);
1134         mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1135         if (termios->c_cflag & CRTSCTS) {
1136                 mr |= UART_MR1_CTS_CTL;
1137                 mr |= UART_MR1_RX_RDY_CTL;
1138         }
1139         msm_write(port, mr, UART_MR1);
1140
1141         /* Configure status bits to ignore based on termio flags. */
1142         port->read_status_mask = 0;
1143         if (termios->c_iflag & INPCK)
1144                 port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1145         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1146                 port->read_status_mask |= UART_SR_RX_BREAK;
1147
1148         uart_update_timeout(port, termios->c_cflag, baud);
1149
1150         /* Try to use DMA */
1151         msm_start_rx_dma(msm_port);
1152
1153         spin_unlock_irqrestore(&port->lock, flags);
1154 }
1155
1156 static const char *msm_type(struct uart_port *port)
1157 {
1158         return "MSM";
1159 }
1160
1161 static void msm_release_port(struct uart_port *port)
1162 {
1163         struct platform_device *pdev = to_platform_device(port->dev);
1164         struct resource *uart_resource;
1165         resource_size_t size;
1166
1167         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1168         if (unlikely(!uart_resource))
1169                 return;
1170         size = resource_size(uart_resource);
1171
1172         release_mem_region(port->mapbase, size);
1173         iounmap(port->membase);
1174         port->membase = NULL;
1175 }
1176
1177 static int msm_request_port(struct uart_port *port)
1178 {
1179         struct platform_device *pdev = to_platform_device(port->dev);
1180         struct resource *uart_resource;
1181         resource_size_t size;
1182         int ret;
1183
1184         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1185         if (unlikely(!uart_resource))
1186                 return -ENXIO;
1187
1188         size = resource_size(uart_resource);
1189
1190         if (!request_mem_region(port->mapbase, size, "msm_serial"))
1191                 return -EBUSY;
1192
1193         port->membase = ioremap(port->mapbase, size);
1194         if (!port->membase) {
1195                 ret = -EBUSY;
1196                 goto fail_release_port;
1197         }
1198
1199         return 0;
1200
1201 fail_release_port:
1202         release_mem_region(port->mapbase, size);
1203         return ret;
1204 }
1205
1206 static void msm_config_port(struct uart_port *port, int flags)
1207 {
1208         int ret;
1209
1210         if (flags & UART_CONFIG_TYPE) {
1211                 port->type = PORT_MSM;
1212                 ret = msm_request_port(port);
1213                 if (ret)
1214                         return;
1215         }
1216 }
1217
1218 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1219 {
1220         if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1221                 return -EINVAL;
1222         if (unlikely(port->irq != ser->irq))
1223                 return -EINVAL;
1224         return 0;
1225 }
1226
1227 static void msm_power(struct uart_port *port, unsigned int state,
1228                       unsigned int oldstate)
1229 {
1230         struct msm_port *msm_port = UART_TO_MSM(port);
1231
1232         switch (state) {
1233         case 0:
1234                 clk_prepare_enable(msm_port->clk);
1235                 clk_prepare_enable(msm_port->pclk);
1236                 break;
1237         case 3:
1238                 clk_disable_unprepare(msm_port->clk);
1239                 clk_disable_unprepare(msm_port->pclk);
1240                 break;
1241         default:
1242                 pr_err("msm_serial: Unknown PM state %d\n", state);
1243         }
1244 }
1245
1246 #ifdef CONFIG_CONSOLE_POLL
1247 static int msm_poll_get_char_single(struct uart_port *port)
1248 {
1249         struct msm_port *msm_port = UART_TO_MSM(port);
1250         unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1251
1252         if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1253                 return NO_POLL_CHAR;
1254
1255         return msm_read(port, rf_reg) & 0xff;
1256 }
1257
1258 static int msm_poll_get_char_dm(struct uart_port *port)
1259 {
1260         int c;
1261         static u32 slop;
1262         static int count;
1263         unsigned char *sp = (unsigned char *)&slop;
1264
1265         /* Check if a previous read had more than one char */
1266         if (count) {
1267                 c = sp[sizeof(slop) - count];
1268                 count--;
1269         /* Or if FIFO is empty */
1270         } else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1271                 /*
1272                  * If RX packing buffer has less than a word, force stale to
1273                  * push contents into RX FIFO
1274                  */
1275                 count = msm_read(port, UARTDM_RXFS);
1276                 count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1277                 if (count) {
1278                         msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1279                         slop = msm_read(port, UARTDM_RF);
1280                         c = sp[0];
1281                         count--;
1282                         msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1283                         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1284                         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1285                                   UART_CR);
1286                 } else {
1287                         c = NO_POLL_CHAR;
1288                 }
1289         /* FIFO has a word */
1290         } else {
1291                 slop = msm_read(port, UARTDM_RF);
1292                 c = sp[0];
1293                 count = sizeof(slop) - 1;
1294         }
1295
1296         return c;
1297 }
1298
1299 static int msm_poll_get_char(struct uart_port *port)
1300 {
1301         u32 imr;
1302         int c;
1303         struct msm_port *msm_port = UART_TO_MSM(port);
1304
1305         /* Disable all interrupts */
1306         imr = msm_read(port, UART_IMR);
1307         msm_write(port, 0, UART_IMR);
1308
1309         if (msm_port->is_uartdm)
1310                 c = msm_poll_get_char_dm(port);
1311         else
1312                 c = msm_poll_get_char_single(port);
1313
1314         /* Enable interrupts */
1315         msm_write(port, imr, UART_IMR);
1316
1317         return c;
1318 }
1319
1320 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1321 {
1322         u32 imr;
1323         struct msm_port *msm_port = UART_TO_MSM(port);
1324
1325         /* Disable all interrupts */
1326         imr = msm_read(port, UART_IMR);
1327         msm_write(port, 0, UART_IMR);
1328
1329         if (msm_port->is_uartdm)
1330                 msm_reset_dm_count(port, 1);
1331
1332         /* Wait until FIFO is empty */
1333         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1334                 cpu_relax();
1335
1336         /* Write a character */
1337         msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1338
1339         /* Wait until FIFO is empty */
1340         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1341                 cpu_relax();
1342
1343         /* Enable interrupts */
1344         msm_write(port, imr, UART_IMR);
1345 }
1346 #endif
1347
1348 static struct uart_ops msm_uart_pops = {
1349         .tx_empty = msm_tx_empty,
1350         .set_mctrl = msm_set_mctrl,
1351         .get_mctrl = msm_get_mctrl,
1352         .stop_tx = msm_stop_tx,
1353         .start_tx = msm_start_tx,
1354         .stop_rx = msm_stop_rx,
1355         .enable_ms = msm_enable_ms,
1356         .break_ctl = msm_break_ctl,
1357         .startup = msm_startup,
1358         .shutdown = msm_shutdown,
1359         .set_termios = msm_set_termios,
1360         .type = msm_type,
1361         .release_port = msm_release_port,
1362         .request_port = msm_request_port,
1363         .config_port = msm_config_port,
1364         .verify_port = msm_verify_port,
1365         .pm = msm_power,
1366 #ifdef CONFIG_CONSOLE_POLL
1367         .poll_get_char  = msm_poll_get_char,
1368         .poll_put_char  = msm_poll_put_char,
1369 #endif
1370 };
1371
1372 static struct msm_port msm_uart_ports[] = {
1373         {
1374                 .uart = {
1375                         .iotype = UPIO_MEM,
1376                         .ops = &msm_uart_pops,
1377                         .flags = UPF_BOOT_AUTOCONF,
1378                         .fifosize = 64,
1379                         .line = 0,
1380                 },
1381         },
1382         {
1383                 .uart = {
1384                         .iotype = UPIO_MEM,
1385                         .ops = &msm_uart_pops,
1386                         .flags = UPF_BOOT_AUTOCONF,
1387                         .fifosize = 64,
1388                         .line = 1,
1389                 },
1390         },
1391         {
1392                 .uart = {
1393                         .iotype = UPIO_MEM,
1394                         .ops = &msm_uart_pops,
1395                         .flags = UPF_BOOT_AUTOCONF,
1396                         .fifosize = 64,
1397                         .line = 2,
1398                 },
1399         },
1400 };
1401
1402 #define UART_NR ARRAY_SIZE(msm_uart_ports)
1403
1404 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1405 {
1406         return &msm_uart_ports[line].uart;
1407 }
1408
1409 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1410 static void __msm_console_write(struct uart_port *port, const char *s,
1411                                 unsigned int count, bool is_uartdm)
1412 {
1413         int i;
1414         int num_newlines = 0;
1415         bool replaced = false;
1416         void __iomem *tf;
1417         int locked = 1;
1418
1419         if (is_uartdm)
1420                 tf = port->membase + UARTDM_TF;
1421         else
1422                 tf = port->membase + UART_TF;
1423
1424         /* Account for newlines that will get a carriage return added */
1425         for (i = 0; i < count; i++)
1426                 if (s[i] == '\n')
1427                         num_newlines++;
1428         count += num_newlines;
1429
1430         if (port->sysrq)
1431                 locked = 0;
1432         else if (oops_in_progress)
1433                 locked = spin_trylock(&port->lock);
1434         else
1435                 spin_lock(&port->lock);
1436
1437         if (is_uartdm)
1438                 msm_reset_dm_count(port, count);
1439
1440         i = 0;
1441         while (i < count) {
1442                 int j;
1443                 unsigned int num_chars;
1444                 char buf[4] = { 0 };
1445
1446                 if (is_uartdm)
1447                         num_chars = min(count - i, (unsigned int)sizeof(buf));
1448                 else
1449                         num_chars = 1;
1450
1451                 for (j = 0; j < num_chars; j++) {
1452                         char c = *s;
1453
1454                         if (c == '\n' && !replaced) {
1455                                 buf[j] = '\r';
1456                                 j++;
1457                                 replaced = true;
1458                         }
1459                         if (j < num_chars) {
1460                                 buf[j] = c;
1461                                 s++;
1462                                 replaced = false;
1463                         }
1464                 }
1465
1466                 while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1467                         cpu_relax();
1468
1469                 iowrite32_rep(tf, buf, 1);
1470                 i += num_chars;
1471         }
1472
1473         if (locked)
1474                 spin_unlock(&port->lock);
1475 }
1476
1477 static void msm_console_write(struct console *co, const char *s,
1478                               unsigned int count)
1479 {
1480         struct uart_port *port;
1481         struct msm_port *msm_port;
1482
1483         BUG_ON(co->index < 0 || co->index >= UART_NR);
1484
1485         port = msm_get_port_from_line(co->index);
1486         msm_port = UART_TO_MSM(port);
1487
1488         __msm_console_write(port, s, count, msm_port->is_uartdm);
1489 }
1490
1491 static int __init msm_console_setup(struct console *co, char *options)
1492 {
1493         struct uart_port *port;
1494         int baud = 115200;
1495         int bits = 8;
1496         int parity = 'n';
1497         int flow = 'n';
1498
1499         if (unlikely(co->index >= UART_NR || co->index < 0))
1500                 return -ENXIO;
1501
1502         port = msm_get_port_from_line(co->index);
1503
1504         if (unlikely(!port->membase))
1505                 return -ENXIO;
1506
1507         msm_init_clock(port);
1508
1509         if (options)
1510                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1511
1512         pr_info("msm_serial: console setup on port #%d\n", port->line);
1513
1514         return uart_set_options(port, co, baud, parity, bits, flow);
1515 }
1516
1517 static void
1518 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1519 {
1520         struct earlycon_device *dev = con->data;
1521
1522         __msm_console_write(&dev->port, s, n, false);
1523 }
1524
1525 static int __init
1526 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1527 {
1528         if (!device->port.membase)
1529                 return -ENODEV;
1530
1531         device->con->write = msm_serial_early_write;
1532         return 0;
1533 }
1534 EARLYCON_DECLARE(msm_serial, msm_serial_early_console_setup);
1535 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1536                     msm_serial_early_console_setup);
1537
1538 static void
1539 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1540 {
1541         struct earlycon_device *dev = con->data;
1542
1543         __msm_console_write(&dev->port, s, n, true);
1544 }
1545
1546 static int __init
1547 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1548                                   const char *opt)
1549 {
1550         if (!device->port.membase)
1551                 return -ENODEV;
1552
1553         device->con->write = msm_serial_early_write_dm;
1554         return 0;
1555 }
1556 EARLYCON_DECLARE(msm_serial_dm, msm_serial_early_console_setup_dm);
1557 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1558                     msm_serial_early_console_setup_dm);
1559
1560 static struct uart_driver msm_uart_driver;
1561
1562 static struct console msm_console = {
1563         .name = "ttyMSM",
1564         .write = msm_console_write,
1565         .device = uart_console_device,
1566         .setup = msm_console_setup,
1567         .flags = CON_PRINTBUFFER,
1568         .index = -1,
1569         .data = &msm_uart_driver,
1570 };
1571
1572 #define MSM_CONSOLE     (&msm_console)
1573
1574 #else
1575 #define MSM_CONSOLE     NULL
1576 #endif
1577
1578 static struct uart_driver msm_uart_driver = {
1579         .owner = THIS_MODULE,
1580         .driver_name = "msm_serial",
1581         .dev_name = "ttyMSM",
1582         .nr = UART_NR,
1583         .cons = MSM_CONSOLE,
1584 };
1585
1586 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1587
1588 static const struct of_device_id msm_uartdm_table[] = {
1589         { .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1590         { .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1591         { .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1592         { .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1593         { }
1594 };
1595
1596 static int msm_serial_probe(struct platform_device *pdev)
1597 {
1598         struct msm_port *msm_port;
1599         struct resource *resource;
1600         struct uart_port *port;
1601         const struct of_device_id *id;
1602         int irq, line;
1603
1604         if (pdev->dev.of_node)
1605                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1606         else
1607                 line = pdev->id;
1608
1609         if (line < 0)
1610                 line = atomic_inc_return(&msm_uart_next_id) - 1;
1611
1612         if (unlikely(line < 0 || line >= UART_NR))
1613                 return -ENXIO;
1614
1615         dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1616
1617         port = msm_get_port_from_line(line);
1618         port->dev = &pdev->dev;
1619         msm_port = UART_TO_MSM(port);
1620
1621         id = of_match_device(msm_uartdm_table, &pdev->dev);
1622         if (id)
1623                 msm_port->is_uartdm = (unsigned long)id->data;
1624         else
1625                 msm_port->is_uartdm = 0;
1626
1627         msm_port->clk = devm_clk_get(&pdev->dev, "core");
1628         if (IS_ERR(msm_port->clk))
1629                 return PTR_ERR(msm_port->clk);
1630
1631         if (msm_port->is_uartdm) {
1632                 msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1633                 if (IS_ERR(msm_port->pclk))
1634                         return PTR_ERR(msm_port->pclk);
1635
1636                 clk_set_rate(msm_port->clk, 1843200);
1637         }
1638
1639         port->uartclk = clk_get_rate(msm_port->clk);
1640         dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1641
1642         resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1643         if (unlikely(!resource))
1644                 return -ENXIO;
1645         port->mapbase = resource->start;
1646
1647         irq = platform_get_irq(pdev, 0);
1648         if (unlikely(irq < 0))
1649                 return -ENXIO;
1650         port->irq = irq;
1651
1652         platform_set_drvdata(pdev, port);
1653
1654         return uart_add_one_port(&msm_uart_driver, port);
1655 }
1656
1657 static int msm_serial_remove(struct platform_device *pdev)
1658 {
1659         struct uart_port *port = platform_get_drvdata(pdev);
1660
1661         uart_remove_one_port(&msm_uart_driver, port);
1662
1663         return 0;
1664 }
1665
1666 static const struct of_device_id msm_match_table[] = {
1667         { .compatible = "qcom,msm-uart" },
1668         { .compatible = "qcom,msm-uartdm" },
1669         {}
1670 };
1671 MODULE_DEVICE_TABLE(of, msm_match_table);
1672
1673 static struct platform_driver msm_platform_driver = {
1674         .remove = msm_serial_remove,
1675         .probe = msm_serial_probe,
1676         .driver = {
1677                 .name = "msm_serial",
1678                 .of_match_table = msm_match_table,
1679         },
1680 };
1681
1682 static int __init msm_serial_init(void)
1683 {
1684         int ret;
1685
1686         ret = uart_register_driver(&msm_uart_driver);
1687         if (unlikely(ret))
1688                 return ret;
1689
1690         ret = platform_driver_register(&msm_platform_driver);
1691         if (unlikely(ret))
1692                 uart_unregister_driver(&msm_uart_driver);
1693
1694         pr_info("msm_serial: driver initialized\n");
1695
1696         return ret;
1697 }
1698
1699 static void __exit msm_serial_exit(void)
1700 {
1701         platform_driver_unregister(&msm_platform_driver);
1702         uart_unregister_driver(&msm_uart_driver);
1703 }
1704
1705 module_init(msm_serial_init);
1706 module_exit(msm_serial_exit);
1707
1708 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1709 MODULE_DESCRIPTION("Driver for msm7x serial device");
1710 MODULE_LICENSE("GPL");