drivers/mmc/host/sdhci.c

   1 /*
   2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   3  *
   4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or (at
   9  * your option) any later version.
  10  *
  11  * Thanks to the following companies for their support:
  12  *
  13  *     - JMicron (hardware and technical support)
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/ktime.h>
  18 #include <linux/highmem.h>
  19 #include <linux/io.h>
  20 #include <linux/module.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sizes.h>
  25 #include <linux/swiotlb.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/of.h>
  29
  30 #include <linux/leds.h>
  31
  32 #include <linux/mmc/mmc.h>
  33 #include <linux/mmc/host.h>
  34 #include <linux/mmc/card.h>
  35 #include <linux/mmc/sdio.h>
  36 #include <linux/mmc/slot-gpio.h>
  37
  38 #include "sdhci.h"
  39
  40 #define DRIVER_NAME "sdhci"
  41
  42 #define DBG(f, x...) \
  43         pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45 #define SDHCI_DUMP(f, x...) \
  46         pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  47
  48 #define MAX_TUNING_LOOP 40
  49
  50 static unsigned int debug_quirks = 0;
  51 static unsigned int debug_quirks2;
  52
  53 static void sdhci_finish_data(struct sdhci_host *);
  54
  55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  56
  57 void sdhci_dumpregs(struct sdhci_host *host)
  58 {
  59         SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  60
  61         SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  62                    sdhci_readl(host, SDHCI_DMA_ADDRESS),
  63                    sdhci_readw(host, SDHCI_HOST_VERSION));
  64         SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  65                    sdhci_readw(host, SDHCI_BLOCK_SIZE),
  66                    sdhci_readw(host, SDHCI_BLOCK_COUNT));
  67         SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  68                    sdhci_readl(host, SDHCI_ARGUMENT),
  69                    sdhci_readw(host, SDHCI_TRANSFER_MODE));
  70         SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  71                    sdhci_readl(host, SDHCI_PRESENT_STATE),
  72                    sdhci_readb(host, SDHCI_HOST_CONTROL));
  73         SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  74                    sdhci_readb(host, SDHCI_POWER_CONTROL),
  75                    sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  76         SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  77                    sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  78                    sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  79         SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  80                    sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  81                    sdhci_readl(host, SDHCI_INT_STATUS));
  82         SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  83                    sdhci_readl(host, SDHCI_INT_ENABLE),
  84                    sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  85         SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n",
  86                    sdhci_readw(host, SDHCI_AUTO_CMD_STATUS),
  87                    sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  88         SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  89                    sdhci_readl(host, SDHCI_CAPABILITIES),
  90                    sdhci_readl(host, SDHCI_CAPABILITIES_1));
  91         SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  92                    sdhci_readw(host, SDHCI_COMMAND),
  93                    sdhci_readl(host, SDHCI_MAX_CURRENT));
  94         SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  95                    sdhci_readl(host, SDHCI_RESPONSE),
  96                    sdhci_readl(host, SDHCI_RESPONSE + 4));
  97         SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  98                    sdhci_readl(host, SDHCI_RESPONSE + 8),
  99                    sdhci_readl(host, SDHCI_RESPONSE + 12));
 100         SDHCI_DUMP("Host ctl2: 0x%08x\n",
 101                    sdhci_readw(host, SDHCI_HOST_CONTROL2));
 102
 103         if (host->flags & SDHCI_USE_ADMA) {
 104                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
 105                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 106                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 107                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 108                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 109                 } else {
 110                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 111                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 112                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 113                 }
 114         }
 115
 116         SDHCI_DUMP("============================================\n");
 117 }
 118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120 /*****************************************************************************\
 121  *                                                                           *
 122  * Low level functions                                                       *
 123  *                                                                           *
 124 \*****************************************************************************/
 125
 126 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 127 {
 128         return cmd->data || cmd->flags & MMC_RSP_BUSY;
 129 }
 130
 131 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 132 {
 133         u32 present;
 134
 135         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 136             !mmc_card_is_removable(host->mmc) || mmc_can_gpio_cd(host->mmc))
 137                 return;
 138
 139         if (enable) {
 140                 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 141                                       SDHCI_CARD_PRESENT;
 142
 143                 host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 144                                        SDHCI_INT_CARD_INSERT;
 145         } else {
 146                 host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 147         }
 148
 149         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 150         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 151 }
 152
 153 static void sdhci_enable_card_detection(struct sdhci_host *host)
 154 {
 155         sdhci_set_card_detection(host, true);
 156 }
 157
 158 static void sdhci_disable_card_detection(struct sdhci_host *host)
 159 {
 160         sdhci_set_card_detection(host, false);
 161 }
 162
 163 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 164 {
 165         if (host->bus_on)
 166                 return;
 167         host->bus_on = true;
 168         pm_runtime_get_noresume(host->mmc->parent);
 169 }
 170
 171 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 172 {
 173         if (!host->bus_on)
 174                 return;
 175         host->bus_on = false;
 176         pm_runtime_put_noidle(host->mmc->parent);
 177 }
 178
 179 void sdhci_reset(struct sdhci_host *host, u8 mask)
 180 {
 181         ktime_t timeout;
 182
 183         sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 184
 185         if (mask & SDHCI_RESET_ALL) {
 186                 host->clock = 0;
 187                 /* Reset-all turns off SD Bus Power */
 188                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 189                         sdhci_runtime_pm_bus_off(host);
 190         }
 191
 192         /* Wait max 100 ms */
 193         timeout = ktime_add_ms(ktime_get(), 100);
 194
 195         /* hw clears the bit when it's done */
 196         while (1) {
 197                 bool timedout = ktime_after(ktime_get(), timeout);
 198
 199                 if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
 200                         break;
 201                 if (timedout) {
 202                         pr_err("%s: Reset 0x%x never completed.\n",
 203                                 mmc_hostname(host->mmc), (int)mask);
 204                         sdhci_dumpregs(host);
 205                         return;
 206                 }
 207                 udelay(10);
 208         }
 209 }
 210 EXPORT_SYMBOL_GPL(sdhci_reset);
 211
 212 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 213 {
 214         if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 215                 struct mmc_host *mmc = host->mmc;
 216
 217                 if (!mmc->ops->get_cd(mmc))
 218                         return;
 219         }
 220
 221         host->ops->reset(host, mask);
 222
 223         if (mask & SDHCI_RESET_ALL) {
 224                 if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 225                         if (host->ops->enable_dma)
 226                                 host->ops->enable_dma(host);
 227                 }
 228
 229                 /* Resetting the controller clears many */
 230                 host->preset_enabled = false;
 231         }
 232 }
 233
 234 static void sdhci_set_default_irqs(struct sdhci_host *host)
 235 {
 236         host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 237                     SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 238                     SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 239                     SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 240                     SDHCI_INT_RESPONSE;
 241
 242         if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 243             host->tuning_mode == SDHCI_TUNING_MODE_3)
 244                 host->ier |= SDHCI_INT_RETUNE;
 245
 246         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 247         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 248 }
 249
 250 static void sdhci_init(struct sdhci_host *host, int soft)
 251 {
 252         struct mmc_host *mmc = host->mmc;
 253
 254         if (soft)
 255                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 256         else
 257                 sdhci_do_reset(host, SDHCI_RESET_ALL);
 258
 259         sdhci_set_default_irqs(host);
 260
 261         host->cqe_on = false;
 262
 263         if (soft) {
 264                 /* force clock reconfiguration */
 265                 host->clock = 0;
 266                 mmc->ops->set_ios(mmc, &mmc->ios);
 267         }
 268 }
 269
 270 static void sdhci_reinit(struct sdhci_host *host)
 271 {
 272         sdhci_init(host, 0);
 273         sdhci_enable_card_detection(host);
 274 }
 275
 276 static void __sdhci_led_activate(struct sdhci_host *host)
 277 {
 278         u8 ctrl;
 279
 280         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 281         ctrl |= SDHCI_CTRL_LED;
 282         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 283 }
 284
 285 static void __sdhci_led_deactivate(struct sdhci_host *host)
 286 {
 287         u8 ctrl;
 288
 289         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 290         ctrl &= ~SDHCI_CTRL_LED;
 291         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 292 }
 293
 294 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
 295 static void sdhci_led_control(struct led_classdev *led,
 296                               enum led_brightness brightness)
 297 {
 298         struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 299         unsigned long flags;
 300
 301         spin_lock_irqsave(&host->lock, flags);
 302
 303         if (host->runtime_suspended)
 304                 goto out;
 305
 306         if (brightness == LED_OFF)
 307                 __sdhci_led_deactivate(host);
 308         else
 309                 __sdhci_led_activate(host);
 310 out:
 311         spin_unlock_irqrestore(&host->lock, flags);
 312 }
 313
 314 static int sdhci_led_register(struct sdhci_host *host)
 315 {
 316         struct mmc_host *mmc = host->mmc;
 317
 318         snprintf(host->led_name, sizeof(host->led_name),
 319                  "%s::", mmc_hostname(mmc));
 320
 321         host->led.name = host->led_name;
 322         host->led.brightness = LED_OFF;
 323         host->led.default_trigger = mmc_hostname(mmc);
 324         host->led.brightness_set = sdhci_led_control;
 325
 326         return led_classdev_register(mmc_dev(mmc), &host->led);
 327 }
 328
 329 static void sdhci_led_unregister(struct sdhci_host *host)
 330 {
 331         led_classdev_unregister(&host->led);
 332 }
 333
 334 static inline void sdhci_led_activate(struct sdhci_host *host)
 335 {
 336 }
 337
 338 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 339 {
 340 }
 341
 342 #else
 343
 344 static inline int sdhci_led_register(struct sdhci_host *host)
 345 {
 346         return 0;
 347 }
 348
 349 static inline void sdhci_led_unregister(struct sdhci_host *host)
 350 {
 351 }
 352
 353 static inline void sdhci_led_activate(struct sdhci_host *host)
 354 {
 355         __sdhci_led_activate(host);
 356 }
 357
 358 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 359 {
 360         __sdhci_led_deactivate(host);
 361 }
 362
 363 #endif
 364
 365 /*****************************************************************************\
 366  *                                                                           *
 367  * Core functions                                                            *
 368  *                                                                           *
 369 \*****************************************************************************/
 370
 371 static void sdhci_read_block_pio(struct sdhci_host *host)
 372 {
 373         unsigned long flags;
 374         size_t blksize, len, chunk;
 375         u32 uninitialized_var(scratch);
 376         u8 *buf;
 377
 378         DBG("PIO reading\n");
 379
 380         blksize = host->data->blksz;
 381         chunk = 0;
 382
 383         local_irq_save(flags);
 384
 385         while (blksize) {
 386                 BUG_ON(!sg_miter_next(&host->sg_miter));
 387
 388                 len = min(host->sg_miter.length, blksize);
 389
 390                 blksize -= len;
 391                 host->sg_miter.consumed = len;
 392
 393                 buf = host->sg_miter.addr;
 394
 395                 while (len) {
 396                         if (chunk == 0) {
 397                                 scratch = sdhci_readl(host, SDHCI_BUFFER);
 398                                 chunk = 4;
 399                         }
 400
 401                         *buf = scratch & 0xFF;
 402
 403                         buf++;
 404                         scratch >>= 8;
 405                         chunk--;
 406                         len--;
 407                 }
 408         }
 409
 410         sg_miter_stop(&host->sg_miter);
 411
 412         local_irq_restore(flags);
 413 }
 414
 415 static void sdhci_write_block_pio(struct sdhci_host *host)
 416 {
 417         unsigned long flags;
 418         size_t blksize, len, chunk;
 419         u32 scratch;
 420         u8 *buf;
 421
 422         DBG("PIO writing\n");
 423
 424         blksize = host->data->blksz;
 425         chunk = 0;
 426         scratch = 0;
 427
 428         local_irq_save(flags);
 429
 430         while (blksize) {
 431                 BUG_ON(!sg_miter_next(&host->sg_miter));
 432
 433                 len = min(host->sg_miter.length, blksize);
 434
 435                 blksize -= len;
 436                 host->sg_miter.consumed = len;
 437
 438                 buf = host->sg_miter.addr;
 439
 440                 while (len) {
 441                         scratch |= (u32)*buf << (chunk * 8);
 442
 443                         buf++;
 444                         chunk++;
 445                         len--;
 446
 447                         if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 448                                 sdhci_writel(host, scratch, SDHCI_BUFFER);
 449                                 chunk = 0;
 450                                 scratch = 0;
 451                         }
 452                 }
 453         }
 454
 455         sg_miter_stop(&host->sg_miter);
 456
 457         local_irq_restore(flags);
 458 }
 459
 460 static void sdhci_transfer_pio(struct sdhci_host *host)
 461 {
 462         u32 mask;
 463
 464         if (host->blocks == 0)
 465                 return;
 466
 467         if (host->data->flags & MMC_DATA_READ)
 468                 mask = SDHCI_DATA_AVAILABLE;
 469         else
 470                 mask = SDHCI_SPACE_AVAILABLE;
 471
 472         /*
 473          * Some controllers (JMicron JMB38x) mess up the buffer bits
 474          * for transfers < 4 bytes. As long as it is just one block,
 475          * we can ignore the bits.
 476          */
 477         if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 478                 (host->data->blocks == 1))
 479                 mask = ~0;
 480
 481         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 482                 if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 483                         udelay(100);
 484
 485                 if (host->data->flags & MMC_DATA_READ)
 486                         sdhci_read_block_pio(host);
 487                 else
 488                         sdhci_write_block_pio(host);
 489
 490                 host->blocks--;
 491                 if (host->blocks == 0)
 492                         break;
 493         }
 494
 495         DBG("PIO transfer complete.\n");
 496 }
 497
 498 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 499                                   struct mmc_data *data, int cookie)
 500 {
 501         int sg_count;
 502
 503         /*
 504          * If the data buffers are already mapped, return the previous
 505          * dma_map_sg() result.
 506          */
 507         if (data->host_cookie == COOKIE_PRE_MAPPED)
 508                 return data->sg_count;
 509
 510         /* Bounce write requests to the bounce buffer */
 511         if (host->bounce_buffer) {
 512                 unsigned int length = data->blksz * data->blocks;
 513
 514                 if (length > host->bounce_buffer_size) {
 515                         pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 516                                mmc_hostname(host->mmc), length,
 517                                host->bounce_buffer_size);
 518                         return -EIO;
 519                 }
 520                 if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 521                         /* Copy the data to the bounce buffer */
 522                         sg_copy_to_buffer(data->sg, data->sg_len,
 523                                           host->bounce_buffer,
 524                                           length);
 525                 }
 526                 /* Switch ownership to the DMA */
 527                 dma_sync_single_for_device(host->mmc->parent,
 528                                            host->bounce_addr,
 529                                            host->bounce_buffer_size,
 530                                            mmc_get_dma_dir(data));
 531                 /* Just a dummy value */
 532                 sg_count = 1;
 533         } else {
 534                 /* Just access the data directly from memory */
 535                 sg_count = dma_map_sg(mmc_dev(host->mmc),
 536                                       data->sg, data->sg_len,
 537                                       mmc_get_dma_dir(data));
 538         }
 539
 540         if (sg_count == 0)
 541                 return -ENOSPC;
 542
 543         data->sg_count = sg_count;
 544         data->host_cookie = cookie;
 545
 546         return sg_count;
 547 }
 548
 549 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 550 {
 551         local_irq_save(*flags);
 552         return kmap_atomic(sg_page(sg)) + sg->offset;
 553 }
 554
 555 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 556 {
 557         kunmap_atomic(buffer);
 558         local_irq_restore(*flags);
 559 }
 560
 561 static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
 562                                   dma_addr_t addr, int len, unsigned cmd)
 563 {
 564         struct sdhci_adma2_64_desc *dma_desc = desc;
 565
 566         /* 32-bit and 64-bit descriptors have these members in same position */
 567         dma_desc->cmd = cpu_to_le16(cmd);
 568         dma_desc->len = cpu_to_le16(len);
 569         dma_desc->addr_lo = cpu_to_le32((u32)addr);
 570
 571         if (host->flags & SDHCI_USE_64_BIT_DMA)
 572                 dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
 573 }
 574
 575 static void sdhci_adma_mark_end(void *desc)
 576 {
 577         struct sdhci_adma2_64_desc *dma_desc = desc;
 578
 579         /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 580         dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 581 }
 582
 583 static void sdhci_adma_table_pre(struct sdhci_host *host,
 584         struct mmc_data *data, int sg_count)
 585 {
 586         struct scatterlist *sg;
 587         unsigned long flags;
 588         dma_addr_t addr, align_addr;
 589         void *desc, *align;
 590         char *buffer;
 591         int len, offset, i;
 592
 593         /*
 594          * The spec does not specify endianness of descriptor table.
 595          * We currently guess that it is LE.
 596          */
 597
 598         host->sg_count = sg_count;
 599
 600         desc = host->adma_table;
 601         align = host->align_buffer;
 602
 603         align_addr = host->align_addr;
 604
 605         for_each_sg(data->sg, sg, host->sg_count, i) {
 606                 addr = sg_dma_address(sg);
 607                 len = sg_dma_len(sg);
 608
 609                 /*
 610                  * The SDHCI specification states that ADMA addresses must
 611                  * be 32-bit aligned. If they aren't, then we use a bounce
 612                  * buffer for the (up to three) bytes that screw up the
 613                  * alignment.
 614                  */
 615                 offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 616                          SDHCI_ADMA2_MASK;
 617                 if (offset) {
 618                         if (data->flags & MMC_DATA_WRITE) {
 619                                 buffer = sdhci_kmap_atomic(sg, &flags);
 620                                 memcpy(align, buffer, offset);
 621                                 sdhci_kunmap_atomic(buffer, &flags);
 622                         }
 623
 624                         /* tran, valid */
 625                         sdhci_adma_write_desc(host, desc, align_addr, offset,
 626                                               ADMA2_TRAN_VALID);
 627
 628                         BUG_ON(offset > 65536);
 629
 630                         align += SDHCI_ADMA2_ALIGN;
 631                         align_addr += SDHCI_ADMA2_ALIGN;
 632
 633                         desc += host->desc_sz;
 634
 635                         addr += offset;
 636                         len -= offset;
 637                 }
 638
 639                 BUG_ON(len > 65536);
 640
 641                 if (len) {
 642                         /* tran, valid */
 643                         sdhci_adma_write_desc(host, desc, addr, len,
 644                                               ADMA2_TRAN_VALID);
 645                         desc += host->desc_sz;
 646                 }
 647
 648                 /*
 649                  * If this triggers then we have a calculation bug
 650                  * somewhere. :/
 651                  */
 652                 WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 653         }
 654
 655         if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 656                 /* Mark the last descriptor as the terminating descriptor */
 657                 if (desc != host->adma_table) {
 658                         desc -= host->desc_sz;
 659                         sdhci_adma_mark_end(desc);
 660                 }
 661         } else {
 662                 /* Add a terminating entry - nop, end, valid */
 663                 sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
 664         }
 665 }
 666
 667 static void sdhci_adma_table_post(struct sdhci_host *host,
 668         struct mmc_data *data)
 669 {
 670         struct scatterlist *sg;
 671         int i, size;
 672         void *align;
 673         char *buffer;
 674         unsigned long flags;
 675
 676         if (data->flags & MMC_DATA_READ) {
 677                 bool has_unaligned = false;
 678
 679                 /* Do a quick scan of the SG list for any unaligned mappings */
 680                 for_each_sg(data->sg, sg, host->sg_count, i)
 681                         if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 682                                 has_unaligned = true;
 683                                 break;
 684                         }
 685
 686                 if (has_unaligned) {
 687                         dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 688                                             data->sg_len, DMA_FROM_DEVICE);
 689
 690                         align = host->align_buffer;
 691
 692                         for_each_sg(data->sg, sg, host->sg_count, i) {
 693                                 if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 694                                         size = SDHCI_ADMA2_ALIGN -
 695                                                (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 696
 697                                         buffer = sdhci_kmap_atomic(sg, &flags);
 698                                         memcpy(buffer, align, size);
 699                                         sdhci_kunmap_atomic(buffer, &flags);
 700
 701                                         align += SDHCI_ADMA2_ALIGN;
 702                                 }
 703                         }
 704                 }
 705         }
 706 }
 707
 708 static u32 sdhci_sdma_address(struct sdhci_host *host)
 709 {
 710         if (host->bounce_buffer)
 711                 return host->bounce_addr;
 712         else
 713                 return sg_dma_address(host->data->sg);
 714 }
 715
 716 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 717 {
 718         u8 count;
 719         struct mmc_data *data = cmd->data;
 720         unsigned target_timeout, current_timeout;
 721
 722         /*
 723          * If the host controller provides us with an incorrect timeout
 724          * value, just skip the check and use 0xE.  The hardware may take
 725          * longer to time out, but that's much better than having a too-short
 726          * timeout value.
 727          */
 728         if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 729                 return 0xE;
 730
 731         /* Unspecified timeout, assume max */
 732         if (!data && !cmd->busy_timeout)
 733                 return 0xE;
 734
 735         /* timeout in us */
 736         if (!data)
 737                 target_timeout = cmd->busy_timeout * 1000;
 738         else {
 739                 target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 740                 if (host->clock && data->timeout_clks) {
 741                         unsigned long long val;
 742
 743                         /*
 744                          * data->timeout_clks is in units of clock cycles.
 745                          * host->clock is in Hz.  target_timeout is in us.
 746                          * Hence, us = 1000000 * cycles / Hz.  Round up.
 747                          */
 748                         val = 1000000ULL * data->timeout_clks;
 749                         if (do_div(val, host->clock))
 750                                 target_timeout++;
 751                         target_timeout += val;
 752                 }
 753         }
 754
 755         /*
 756          * Figure out needed cycles.
 757          * We do this in steps in order to fit inside a 32 bit int.
 758          * The first step is the minimum timeout, which will have a
 759          * minimum resolution of 6 bits:
 760          * (1) 2^13*1000 > 2^22,
 761          * (2) host->timeout_clk < 2^16
 762          *     =>
 763          *     (1) / (2) > 2^6
 764          */
 765         count = 0;
 766         current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 767         while (current_timeout < target_timeout) {
 768                 count++;
 769                 current_timeout <<= 1;
 770                 if (count >= 0xF)
 771                         break;
 772         }
 773
 774         if (count >= 0xF) {
 775                 DBG("Too large timeout 0x%x requested for CMD%d!\n",
 776                     count, cmd->opcode);
 777                 count = 0xE;
 778         }
 779
 780         return count;
 781 }
 782
 783 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 784 {
 785         u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 786         u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 787
 788         if (host->flags & SDHCI_REQ_USE_DMA)
 789                 host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 790         else
 791                 host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 792
 793         if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))
 794                 host->ier |= SDHCI_INT_AUTO_CMD_ERR;
 795         else
 796                 host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;
 797
 798         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 799         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 800 }
 801
 802 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 803 {
 804         u8 count;
 805
 806         if (host->ops->set_timeout) {
 807                 host->ops->set_timeout(host, cmd);
 808         } else {
 809                 count = sdhci_calc_timeout(host, cmd);
 810                 sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 811         }
 812 }
 813
 814 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 815 {
 816         u8 ctrl;
 817         struct mmc_data *data = cmd->data;
 818
 819         if (sdhci_data_line_cmd(cmd))
 820                 sdhci_set_timeout(host, cmd);
 821
 822         if (!data)
 823                 return;
 824
 825         WARN_ON(host->data);
 826
 827         /* Sanity checks */
 828         BUG_ON(data->blksz * data->blocks > 524288);
 829         BUG_ON(data->blksz > host->mmc->max_blk_size);
 830         BUG_ON(data->blocks > 65535);
 831
 832         host->data = data;
 833         host->data_early = 0;
 834         host->data->bytes_xfered = 0;
 835
 836         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 837                 struct scatterlist *sg;
 838                 unsigned int length_mask, offset_mask;
 839                 int i;
 840
 841                 host->flags |= SDHCI_REQ_USE_DMA;
 842
 843                 /*
 844                  * FIXME: This doesn't account for merging when mapping the
 845                  * scatterlist.
 846                  *
 847                  * The assumption here being that alignment and lengths are
 848                  * the same after DMA mapping to device address space.
 849                  */
 850                 length_mask = 0;
 851                 offset_mask = 0;
 852                 if (host->flags & SDHCI_USE_ADMA) {
 853                         if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
 854                                 length_mask = 3;
 855                                 /*
 856                                  * As we use up to 3 byte chunks to work
 857                                  * around alignment problems, we need to
 858                                  * check the offset as well.
 859                                  */
 860                                 offset_mask = 3;
 861                         }
 862                 } else {
 863                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
 864                                 length_mask = 3;
 865                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
 866                                 offset_mask = 3;
 867                 }
 868
 869                 if (unlikely(length_mask | offset_mask)) {
 870                         for_each_sg(data->sg, sg, data->sg_len, i) {
 871                                 if (sg->length & length_mask) {
 872                                         DBG("Reverting to PIO because of transfer size (%d)\n",
 873                                             sg->length);
 874                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 875                                         break;
 876                                 }
 877                                 if (sg->offset & offset_mask) {
 878                                         DBG("Reverting to PIO because of bad alignment\n");
 879                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 880                                         break;
 881                                 }
 882                         }
 883                 }
 884         }
 885
 886         if (host->flags & SDHCI_REQ_USE_DMA) {
 887                 int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
 888
 889                 if (sg_cnt <= 0) {
 890                         /*
 891                          * This only happens when someone fed
 892                          * us an invalid request.
 893                          */
 894                         WARN_ON(1);
 895                         host->flags &= ~SDHCI_REQ_USE_DMA;
 896                 } else if (host->flags & SDHCI_USE_ADMA) {
 897                         sdhci_adma_table_pre(host, data, sg_cnt);
 898
 899                         sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
 900                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 901                                 sdhci_writel(host,
 902                                              (u64)host->adma_addr >> 32,
 903                                              SDHCI_ADMA_ADDRESS_HI);
 904                 } else {
 905                         WARN_ON(sg_cnt != 1);
 906                         sdhci_writel(host, sdhci_sdma_address(host),
 907                                      SDHCI_DMA_ADDRESS);
 908                 }
 909         }
 910
 911         /*
 912          * Always adjust the DMA selection as some controllers
 913          * (e.g. JMicron) can't do PIO properly when the selection
 914          * is ADMA.
 915          */
 916         if (host->version >= SDHCI_SPEC_200) {
 917                 ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 918                 ctrl &= ~SDHCI_CTRL_DMA_MASK;
 919                 if ((host->flags & SDHCI_REQ_USE_DMA) &&
 920                         (host->flags & SDHCI_USE_ADMA)) {
 921                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 922                                 ctrl |= SDHCI_CTRL_ADMA64;
 923                         else
 924                                 ctrl |= SDHCI_CTRL_ADMA32;
 925                 } else {
 926                         ctrl |= SDHCI_CTRL_SDMA;
 927                 }
 928                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 929         }
 930
 931         if (!(host->flags & SDHCI_REQ_USE_DMA)) {
 932                 int flags;
 933
 934                 flags = SG_MITER_ATOMIC;
 935                 if (host->data->flags & MMC_DATA_READ)
 936                         flags |= SG_MITER_TO_SG;
 937                 else
 938                         flags |= SG_MITER_FROM_SG;
 939                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 940                 host->blocks = data->blocks;
 941         }
 942
 943         sdhci_set_transfer_irqs(host);
 944
 945         /* Set the DMA boundary value and block size */
 946         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
 947                      SDHCI_BLOCK_SIZE);
 948         sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
 949 }
 950
 951 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
 952                                     struct mmc_request *mrq)
 953 {
 954         return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
 955                !mrq->cap_cmd_during_tfr;
 956 }
 957
 958 static void sdhci_set_transfer_mode(struct sdhci_host *host,
 959         struct mmc_command *cmd)
 960 {
 961         u16 mode = 0;
 962         struct mmc_data *data = cmd->data;
 963
 964         if (data == NULL) {
 965                 if (host->quirks2 &
 966                         SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
 967                         sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
 968                 } else {
 969                 /* clear Auto CMD settings for no data CMDs */
 970                         mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
 971                         sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
 972                                 SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
 973                 }
 974                 return;
 975         }
 976
 977         WARN_ON(!host->data);
 978
 979         if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
 980                 mode = SDHCI_TRNS_BLK_CNT_EN;
 981
 982         if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
 983                 mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
 984                 /*
 985                  * If we are sending CMD23, CMD12 never gets sent
 986                  * on successful completion (so no Auto-CMD12).
 987                  */
 988                 if (sdhci_auto_cmd12(host, cmd->mrq) &&
 989                     (cmd->opcode != SD_IO_RW_EXTENDED))
 990                         mode |= SDHCI_TRNS_AUTO_CMD12;
 991                 else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
 992                         mode |= SDHCI_TRNS_AUTO_CMD23;
 993                         sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
 994                 }
 995         }
 996
 997         if (data->flags & MMC_DATA_READ)
 998                 mode |= SDHCI_TRNS_READ;
 999         if (host->flags & SDHCI_REQ_USE_DMA)
1000                 mode |= SDHCI_TRNS_DMA;
1001
1002         sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1003 }
1004
1005 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1006 {
1007         return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1008                 ((mrq->cmd && mrq->cmd->error) ||
1009                  (mrq->sbc && mrq->sbc->error) ||
1010                  (mrq->data && mrq->data->stop && mrq->data->stop->error) ||
1011                  (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1012 }
1013
1014 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1015 {
1016         int i;
1017
1018         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1019                 if (host->mrqs_done[i] == mrq) {
1020                         WARN_ON(1);
1021                         return;
1022                 }
1023         }
1024
1025         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1026                 if (!host->mrqs_done[i]) {
1027                         host->mrqs_done[i] = mrq;
1028                         break;
1029                 }
1030         }
1031
1032         WARN_ON(i >= SDHCI_MAX_MRQS);
1033
1034         tasklet_schedule(&host->finish_tasklet);
1035 }
1036
1037 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1038 {
1039         if (host->cmd && host->cmd->mrq == mrq)
1040                 host->cmd = NULL;
1041
1042         if (host->data_cmd && host->data_cmd->mrq == mrq)
1043                 host->data_cmd = NULL;
1044
1045         if (host->data && host->data->mrq == mrq)
1046                 host->data = NULL;
1047
1048         if (sdhci_needs_reset(host, mrq))
1049                 host->pending_reset = true;
1050
1051         __sdhci_finish_mrq(host, mrq);
1052 }
1053
1054 static void sdhci_finish_data(struct sdhci_host *host)
1055 {
1056         struct mmc_command *data_cmd = host->data_cmd;
1057         struct mmc_data *data = host->data;
1058
1059         host->data = NULL;
1060         host->data_cmd = NULL;
1061
1062         /*
1063          * The controller needs a reset of internal state machines upon error
1064          * conditions.
1065          */
1066         if (data->error) {
1067                 if (!host->cmd || host->cmd == data_cmd)
1068                         sdhci_do_reset(host, SDHCI_RESET_CMD);
1069                 sdhci_do_reset(host, SDHCI_RESET_DATA);
1070         }
1071
1072         if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1073             (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1074                 sdhci_adma_table_post(host, data);
1075
1076         /*
1077          * The specification states that the block count register must
1078          * be updated, but it does not specify at what point in the
1079          * data flow. That makes the register entirely useless to read
1080          * back so we have to assume that nothing made it to the card
1081          * in the event of an error.
1082          */
1083         if (data->error)
1084                 data->bytes_xfered = 0;
1085         else
1086                 data->bytes_xfered = data->blksz * data->blocks;
1087
1088         /*
1089          * Need to send CMD12 if -
1090          * a) open-ended multiblock transfer (no CMD23)
1091          * b) error in multiblock transfer
1092          */
1093         if (data->stop &&
1094             (data->error ||
1095              !data->mrq->sbc)) {
1096                 /*
1097                  * 'cap_cmd_during_tfr' request must not use the command line
1098                  * after mmc_command_done() has been called. It is upper layer's
1099                  * responsibility to send the stop command if required.
1100                  */
1101                 if (data->mrq->cap_cmd_during_tfr) {
1102                         sdhci_finish_mrq(host, data->mrq);
1103                 } else {
1104                         /* Avoid triggering warning in sdhci_send_command() */
1105                         host->cmd = NULL;
1106                         sdhci_send_command(host, data->stop);
1107                 }
1108         } else {
1109                 sdhci_finish_mrq(host, data->mrq);
1110         }
1111 }
1112
1113 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1114                             unsigned long timeout)
1115 {
1116         if (sdhci_data_line_cmd(mrq->cmd))
1117                 mod_timer(&host->data_timer, timeout);
1118         else
1119                 mod_timer(&host->timer, timeout);
1120 }
1121
1122 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1123 {
1124         if (sdhci_data_line_cmd(mrq->cmd))
1125                 del_timer(&host->data_timer);
1126         else
1127                 del_timer(&host->timer);
1128 }
1129
1130 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1131 {
1132         int flags;
1133         u32 mask;
1134         unsigned long timeout;
1135
1136         WARN_ON(host->cmd);
1137
1138         /* Initially, a command has no error */
1139         cmd->error = 0;
1140
1141         if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1142             cmd->opcode == MMC_STOP_TRANSMISSION)
1143                 cmd->flags |= MMC_RSP_BUSY;
1144
1145         /* Wait max 10 ms */
1146         timeout = 10;
1147
1148         mask = SDHCI_CMD_INHIBIT;
1149         if (sdhci_data_line_cmd(cmd))
1150                 mask |= SDHCI_DATA_INHIBIT;
1151
1152         /* We shouldn't wait for data inihibit for stop commands, even
1153            though they might use busy signaling */
1154         if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1155                 mask &= ~SDHCI_DATA_INHIBIT;
1156
1157         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1158                 if (timeout == 0) {
1159                         pr_err("%s: Controller never released inhibit bit(s).\n",
1160                                mmc_hostname(host->mmc));
1161                         sdhci_dumpregs(host);
1162                         cmd->error = -EIO;
1163                         sdhci_finish_mrq(host, cmd->mrq);
1164                         return;
1165                 }
1166                 timeout--;
1167                 mdelay(1);
1168         }
1169
1170         timeout = jiffies;
1171         if (!cmd->data && cmd->busy_timeout > 9000)
1172                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1173         else
1174                 timeout += 10 * HZ;
1175         sdhci_mod_timer(host, cmd->mrq, timeout);
1176
1177         host->cmd = cmd;
1178         if (sdhci_data_line_cmd(cmd)) {
1179                 WARN_ON(host->data_cmd);
1180                 host->data_cmd = cmd;
1181         }
1182
1183         sdhci_prepare_data(host, cmd);
1184
1185         sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1186
1187         sdhci_set_transfer_mode(host, cmd);
1188
1189         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1190                 pr_err("%s: Unsupported response type!\n",
1191                         mmc_hostname(host->mmc));
1192                 cmd->error = -EINVAL;
1193                 sdhci_finish_mrq(host, cmd->mrq);
1194                 return;
1195         }
1196
1197         if (!(cmd->flags & MMC_RSP_PRESENT))
1198                 flags = SDHCI_CMD_RESP_NONE;
1199         else if (cmd->flags & MMC_RSP_136)
1200                 flags = SDHCI_CMD_RESP_LONG;
1201         else if (cmd->flags & MMC_RSP_BUSY)
1202                 flags = SDHCI_CMD_RESP_SHORT_BUSY;
1203         else
1204                 flags = SDHCI_CMD_RESP_SHORT;
1205
1206         if (cmd->flags & MMC_RSP_CRC)
1207                 flags |= SDHCI_CMD_CRC;
1208         if (cmd->flags & MMC_RSP_OPCODE)
1209                 flags |= SDHCI_CMD_INDEX;
1210
1211         /* CMD19 is special in that the Data Present Select should be set */
1212         if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1213             cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1214                 flags |= SDHCI_CMD_DATA;
1215
1216         sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1217 }
1218 EXPORT_SYMBOL_GPL(sdhci_send_command);
1219
1220 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1221 {
1222         int i, reg;
1223
1224         for (i = 0; i < 4; i++) {
1225                 reg = SDHCI_RESPONSE + (3 - i) * 4;
1226                 cmd->resp[i] = sdhci_readl(host, reg);
1227         }
1228
1229         if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1230                 return;
1231
1232         /* CRC is stripped so we need to do some shifting */
1233         for (i = 0; i < 4; i++) {
1234                 cmd->resp[i] <<= 8;
1235                 if (i != 3)
1236                         cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1237         }
1238 }
1239
1240 static void sdhci_finish_command(struct sdhci_host *host)
1241 {
1242         struct mmc_command *cmd = host->cmd;
1243
1244         host->cmd = NULL;
1245
1246         if (cmd->flags & MMC_RSP_PRESENT) {
1247                 if (cmd->flags & MMC_RSP_136) {
1248                         sdhci_read_rsp_136(host, cmd);
1249                 } else {
1250                         cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1251                 }
1252         }
1253
1254         if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1255                 mmc_command_done(host->mmc, cmd->mrq);
1256
1257         /*
1258          * The host can send and interrupt when the busy state has
1259          * ended, allowing us to wait without wasting CPU cycles.
1260          * The busy signal uses DAT0 so this is similar to waiting
1261          * for data to complete.
1262          *
1263          * Note: The 1.0 specification is a bit ambiguous about this
1264          *       feature so there might be some problems with older
1265          *       controllers.
1266          */
1267         if (cmd->flags & MMC_RSP_BUSY) {
1268                 if (cmd->data) {
1269                         DBG("Cannot wait for busy signal when also doing a data transfer");
1270                 } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1271                            cmd == host->data_cmd) {
1272                         /* Command complete before busy is ended */
1273                         return;
1274                 }
1275         }
1276
1277         /* Finished CMD23, now send actual command. */
1278         if (cmd == cmd->mrq->sbc) {
1279                 sdhci_send_command(host, cmd->mrq->cmd);
1280         } else {
1281
1282                 /* Processed actual command. */
1283                 if (host->data && host->data_early)
1284                         sdhci_finish_data(host);
1285
1286                 if (!cmd->data)
1287                         sdhci_finish_mrq(host, cmd->mrq);
1288         }
1289 }
1290
1291 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1292 {
1293         u16 preset = 0;
1294
1295         switch (host->timing) {
1296         case MMC_TIMING_MMC_HS:
1297         case MMC_TIMING_SD_HS:
1298                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HIGH_SPEED);
1299                 break;
1300         case MMC_TIMING_UHS_SDR12:
1301                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1302                 break;
1303         case MMC_TIMING_UHS_SDR25:
1304                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1305                 break;
1306         case MMC_TIMING_UHS_SDR50:
1307                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1308                 break;
1309         case MMC_TIMING_UHS_SDR104:
1310         case MMC_TIMING_MMC_HS200:
1311                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1312                 break;
1313         case MMC_TIMING_UHS_DDR50:
1314         case MMC_TIMING_MMC_DDR52:
1315                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1316                 break;
1317         case MMC_TIMING_MMC_HS400:
1318                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1319                 break;
1320         default:
1321                 pr_warn("%s: Invalid UHS-I mode selected\n",
1322                         mmc_hostname(host->mmc));
1323                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1324                 break;
1325         }
1326         return preset;
1327 }
1328
1329 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1330                    unsigned int *actual_clock)
1331 {
1332         int div = 0; /* Initialized for compiler warning */
1333         int real_div = div, clk_mul = 1;
1334         u16 clk = 0;
1335         bool switch_base_clk = false;
1336
1337         if (host->version >= SDHCI_SPEC_300) {
1338                 if (host->preset_enabled) {
1339                         u16 pre_val;
1340
1341                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1342                         pre_val = sdhci_get_preset_value(host);
1343                         div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1344                                 >> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1345                         if (host->clk_mul &&
1346                                 (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1347                                 clk = SDHCI_PROG_CLOCK_MODE;
1348                                 real_div = div + 1;
1349                                 clk_mul = host->clk_mul;
1350                         } else {
1351                                 real_div = max_t(int, 1, div << 1);
1352                         }
1353                         goto clock_set;
1354                 }
1355
1356                 /*
1357                  * Check if the Host Controller supports Programmable Clock
1358                  * Mode.
1359                  */
1360                 if (host->clk_mul) {
1361                         for (div = 1; div <= 1024; div++) {
1362                                 if ((host->max_clk * host->clk_mul / div)
1363                                         <= clock)
1364                                         break;
1365                         }
1366                         if ((host->max_clk * host->clk_mul / div) <= clock) {
1367                                 /*
1368                                  * Set Programmable Clock Mode in the Clock
1369                                  * Control register.
1370                                  */
1371                                 clk = SDHCI_PROG_CLOCK_MODE;
1372                                 real_div = div;
1373                                 clk_mul = host->clk_mul;
1374                                 div--;
1375                         } else {
1376                                 /*
1377                                  * Divisor can be too small to reach clock
1378                                  * speed requirement. Then use the base clock.
1379                                  */
1380                                 switch_base_clk = true;
1381                         }
1382                 }
1383
1384                 if (!host->clk_mul || switch_base_clk) {
1385                         /* Version 3.00 divisors must be a multiple of 2. */
1386                         if (host->max_clk <= clock)
1387                                 div = 1;
1388                         else {
1389                                 for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1390                                      div += 2) {
1391                                         if ((host->max_clk / div) <= clock)
1392                                                 break;
1393                                 }
1394                         }
1395                         real_div = div;
1396                         div >>= 1;
1397                         if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1398                                 && !div && host->max_clk <= 25000000)
1399                                 div = 1;
1400                 }
1401         } else {
1402                 /* Version 2.00 divisors must be a power of 2. */
1403                 for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1404                         if ((host->max_clk / div) <= clock)
1405                                 break;
1406                 }
1407                 real_div = div;
1408                 div >>= 1;
1409         }
1410
1411 clock_set:
1412         if (real_div)
1413                 *actual_clock = (host->max_clk * clk_mul) / real_div;
1414         clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1415         clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1416                 << SDHCI_DIVIDER_HI_SHIFT;
1417
1418         return clk;
1419 }
1420 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1421
1422 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1423 {
1424         ktime_t timeout;
1425
1426         clk |= SDHCI_CLOCK_INT_EN;
1427         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1428
1429         /* Wait max 20 ms */
1430         timeout = ktime_add_ms(ktime_get(), 20);
1431         while (1) {
1432                 bool timedout = ktime_after(ktime_get(), timeout);
1433
1434                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1435                 if (clk & SDHCI_CLOCK_INT_STABLE)
1436                         break;
1437                 if (timedout) {
1438                         pr_err("%s: Internal clock never stabilised.\n",
1439                                mmc_hostname(host->mmc));
1440                         sdhci_dumpregs(host);
1441                         return;
1442                 }
1443                 udelay(10);
1444         }
1445
1446         clk |= SDHCI_CLOCK_CARD_EN;
1447         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1448 }
1449 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1450
1451 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1452 {
1453         u16 clk;
1454
1455         host->mmc->actual_clock = 0;
1456
1457         sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1458
1459         if (clock == 0)
1460                 return;
1461
1462         clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1463         sdhci_enable_clk(host, clk);
1464 }
1465 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1466
1467 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1468                                 unsigned short vdd)
1469 {
1470         struct mmc_host *mmc = host->mmc;
1471
1472         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1473
1474         if (mode != MMC_POWER_OFF)
1475                 sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1476         else
1477                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1478 }
1479
1480 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1481                            unsigned short vdd)
1482 {
1483         u8 pwr = 0;
1484
1485         if (mode != MMC_POWER_OFF) {
1486                 switch (1 << vdd) {
1487                 case MMC_VDD_165_195:
1488                 /*
1489                  * Without a regulator, SDHCI does not support 2.0v
1490                  * so we only get here if the driver deliberately
1491                  * added the 2.0v range to ocr_avail. Map it to 1.8v
1492                  * for the purpose of turning on the power.
1493                  */
1494                 case MMC_VDD_20_21:
1495                         pwr = SDHCI_POWER_180;
1496                         break;
1497                 case MMC_VDD_29_30:
1498                 case MMC_VDD_30_31:
1499                         pwr = SDHCI_POWER_300;
1500                         break;
1501                 case MMC_VDD_32_33:
1502                 case MMC_VDD_33_34:
1503                         pwr = SDHCI_POWER_330;
1504                         break;
1505                 default:
1506                         WARN(1, "%s: Invalid vdd %#x\n",
1507                              mmc_hostname(host->mmc), vdd);
1508                         break;
1509                 }
1510         }
1511
1512         if (host->pwr == pwr)
1513                 return;
1514
1515         host->pwr = pwr;
1516
1517         if (pwr == 0) {
1518                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1519                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1520                         sdhci_runtime_pm_bus_off(host);
1521         } else {
1522                 /*
1523                  * Spec says that we should clear the power reg before setting
1524                  * a new value. Some controllers don't seem to like this though.
1525                  */
1526                 if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1527                         sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1528
1529                 /*
1530                  * At least the Marvell CaFe chip gets confused if we set the
1531                  * voltage and set turn on power at the same time, so set the
1532                  * voltage first.
1533                  */
1534                 if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1535                         sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1536
1537                 pwr |= SDHCI_POWER_ON;
1538
1539                 sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1540
1541                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1542                         sdhci_runtime_pm_bus_on(host);
1543
1544                 /*
1545                  * Some controllers need an extra 10ms delay of 10ms before
1546                  * they can apply clock after applying power
1547                  */
1548                 if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1549                         mdelay(10);
1550         }
1551 }
1552 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1553
1554 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1555                      unsigned short vdd)
1556 {
1557         if (IS_ERR(host->mmc->supply.vmmc))
1558                 sdhci_set_power_noreg(host, mode, vdd);
1559         else
1560                 sdhci_set_power_reg(host, mode, vdd);
1561 }
1562 EXPORT_SYMBOL_GPL(sdhci_set_power);
1563
1564 /*****************************************************************************\
1565  *                                                                           *
1566  * MMC callbacks                                                             *
1567  *                                                                           *
1568 \*****************************************************************************/
1569
1570 static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1571 {
1572         struct sdhci_host *host;
1573         int present;
1574         unsigned long flags;
1575
1576         host = mmc_priv(mmc);
1577
1578         /* Firstly check card presence */
1579         present = mmc->ops->get_cd(mmc);
1580
1581         spin_lock_irqsave(&host->lock, flags);
1582
1583         sdhci_led_activate(host);
1584
1585         /*
1586          * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1587          * requests if Auto-CMD12 is enabled.
1588          */
1589         if (sdhci_auto_cmd12(host, mrq)) {
1590                 if (mrq->stop) {
1591                         mrq->data->stop = NULL;
1592                         mrq->stop = NULL;
1593                 }
1594         }
1595
1596         if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1597                 mrq->cmd->error = -ENOMEDIUM;
1598                 sdhci_finish_mrq(host, mrq);
1599         } else {
1600                 if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1601                         sdhci_send_command(host, mrq->sbc);
1602                 else
1603                         sdhci_send_command(host, mrq->cmd);
1604         }
1605
1606         mmiowb();
1607         spin_unlock_irqrestore(&host->lock, flags);
1608 }
1609
1610 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1611 {
1612         u8 ctrl;
1613
1614         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1615         if (width == MMC_BUS_WIDTH_8) {
1616                 ctrl &= ~SDHCI_CTRL_4BITBUS;
1617                 ctrl |= SDHCI_CTRL_8BITBUS;
1618         } else {
1619                 if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1620                         ctrl &= ~SDHCI_CTRL_8BITBUS;
1621                 if (width == MMC_BUS_WIDTH_4)
1622                         ctrl |= SDHCI_CTRL_4BITBUS;
1623                 else
1624                         ctrl &= ~SDHCI_CTRL_4BITBUS;
1625         }
1626         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1627 }
1628 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1629
1630 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1631 {
1632         u16 ctrl_2;
1633
1634         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1635         /* Select Bus Speed Mode for host */
1636         ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1637         if ((timing == MMC_TIMING_MMC_HS200) ||
1638             (timing == MMC_TIMING_UHS_SDR104))
1639                 ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1640         else if (timing == MMC_TIMING_UHS_SDR12)
1641                 ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1642         else if (timing == MMC_TIMING_UHS_SDR25)
1643                 ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1644         else if (timing == MMC_TIMING_UHS_SDR50)
1645                 ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1646         else if ((timing == MMC_TIMING_UHS_DDR50) ||
1647                  (timing == MMC_TIMING_MMC_DDR52))
1648                 ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1649         else if (timing == MMC_TIMING_MMC_HS400)
1650                 ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1651         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1652 }
1653 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1654
1655 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1656 {
1657         struct sdhci_host *host = mmc_priv(mmc);
1658         u8 ctrl;
1659
1660         if (ios->power_mode == MMC_POWER_UNDEFINED)
1661                 return;
1662
1663         if (host->flags & SDHCI_DEVICE_DEAD) {
1664                 if (!IS_ERR(mmc->supply.vmmc) &&
1665                     ios->power_mode == MMC_POWER_OFF)
1666                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1667                 return;
1668         }
1669
1670         /*
1671          * Reset the chip on each power off.
1672          * Should clear out any weird states.
1673          */
1674         if (ios->power_mode == MMC_POWER_OFF) {
1675                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1676                 sdhci_reinit(host);
1677         }
1678
1679         if (host->version >= SDHCI_SPEC_300 &&
1680                 (ios->power_mode == MMC_POWER_UP) &&
1681                 !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1682                 sdhci_enable_preset_value(host, false);
1683
1684         if (!ios->clock || ios->clock != host->clock) {
1685                 host->ops->set_clock(host, ios->clock);
1686                 host->clock = ios->clock;
1687
1688                 if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1689                     host->clock) {
1690                         host->timeout_clk = host->mmc->actual_clock ?
1691                                                 host->mmc->actual_clock / 1000 :
1692                                                 host->clock / 1000;
1693                         host->mmc->max_busy_timeout =
1694                                 host->ops->get_max_timeout_count ?
1695                                 host->ops->get_max_timeout_count(host) :
1696                                 1 << 27;
1697                         host->mmc->max_busy_timeout /= host->timeout_clk;
1698                 }
1699         }
1700
1701         if (host->ops->set_power)
1702                 host->ops->set_power(host, ios->power_mode, ios->vdd);
1703         else
1704                 sdhci_set_power(host, ios->power_mode, ios->vdd);
1705
1706         if (host->ops->platform_send_init_74_clocks)
1707                 host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1708
1709         host->ops->set_bus_width(host, ios->bus_width);
1710
1711         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1712
1713         if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1714                 if (ios->timing == MMC_TIMING_SD_HS ||
1715                      ios->timing == MMC_TIMING_MMC_HS ||
1716                      ios->timing == MMC_TIMING_MMC_HS400 ||
1717                      ios->timing == MMC_TIMING_MMC_HS200 ||
1718                      ios->timing == MMC_TIMING_MMC_DDR52 ||
1719                      ios->timing == MMC_TIMING_UHS_SDR50 ||
1720                      ios->timing == MMC_TIMING_UHS_SDR104 ||
1721                      ios->timing == MMC_TIMING_UHS_DDR50 ||
1722                      ios->timing == MMC_TIMING_UHS_SDR25)
1723                         ctrl |= SDHCI_CTRL_HISPD;
1724                 else
1725                         ctrl &= ~SDHCI_CTRL_HISPD;
1726         }
1727
1728         if (host->version >= SDHCI_SPEC_300) {
1729                 u16 clk, ctrl_2;
1730
1731                 if (!host->preset_enabled) {
1732                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1733                         /*
1734                          * We only need to set Driver Strength if the
1735                          * preset value enable is not set.
1736                          */
1737                         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1738                         ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1739                         if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1740                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1741                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1742                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1743                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1744                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1745                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1746                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1747                         else {
1748                                 pr_warn("%s: invalid driver type, default to driver type B\n",
1749                                         mmc_hostname(mmc));
1750                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1751                         }
1752
1753                         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1754                 } else {
1755                         /*
1756                          * According to SDHC Spec v3.00, if the Preset Value
1757                          * Enable in the Host Control 2 register is set, we
1758                          * need to reset SD Clock Enable before changing High
1759                          * Speed Enable to avoid generating clock gliches.
1760                          */
1761
1762                         /* Reset SD Clock Enable */
1763                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1764                         clk &= ~SDHCI_CLOCK_CARD_EN;
1765                         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1766
1767                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1768
1769                         /* Re-enable SD Clock */
1770                         host->ops->set_clock(host, host->clock);
1771                 }
1772
1773                 /* Reset SD Clock Enable */
1774                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1775                 clk &= ~SDHCI_CLOCK_CARD_EN;
1776                 sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1777
1778                 host->ops->set_uhs_signaling(host, ios->timing);
1779                 host->timing = ios->timing;
1780
1781                 if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1782                                 ((ios->timing == MMC_TIMING_UHS_SDR12) ||
1783                                  (ios->timing == MMC_TIMING_UHS_SDR25) ||
1784                                  (ios->timing == MMC_TIMING_UHS_SDR50) ||
1785                                  (ios->timing == MMC_TIMING_UHS_SDR104) ||
1786                                  (ios->timing == MMC_TIMING_UHS_DDR50) ||
1787                                  (ios->timing == MMC_TIMING_MMC_DDR52))) {
1788                         u16 preset;
1789
1790                         sdhci_enable_preset_value(host, true);
1791                         preset = sdhci_get_preset_value(host);
1792                         ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1793                                 >> SDHCI_PRESET_DRV_SHIFT;
1794                 }
1795
1796                 /* Re-enable SD Clock */
1797                 host->ops->set_clock(host, host->clock);
1798         } else
1799                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1800
1801         /*
1802          * Some (ENE) controllers go apeshit on some ios operation,
1803          * signalling timeout and CRC errors even on CMD0. Resetting
1804          * it on each ios seems to solve the problem.
1805          */
1806         if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
1807                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1808
1809         mmiowb();
1810 }
1811 EXPORT_SYMBOL_GPL(sdhci_set_ios);
1812
1813 static int sdhci_get_cd(struct mmc_host *mmc)
1814 {
1815         struct sdhci_host *host = mmc_priv(mmc);
1816         int gpio_cd = mmc_gpio_get_cd(mmc);
1817
1818         if (host->flags & SDHCI_DEVICE_DEAD)
1819                 return 0;
1820
1821         /* If nonremovable, assume that the card is always present. */
1822         if (!mmc_card_is_removable(host->mmc))
1823                 return 1;
1824
1825         /*
1826          * Try slot gpio detect, if defined it take precedence
1827          * over build in controller functionality
1828          */
1829         if (gpio_cd >= 0)
1830                 return !!gpio_cd;
1831
1832         /* If polling, assume that the card is always present. */
1833         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
1834                 return 1;
1835
1836         /* Host native card detect */
1837         return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1838 }
1839
1840 static int sdhci_check_ro(struct sdhci_host *host)
1841 {
1842         unsigned long flags;
1843         int is_readonly;
1844
1845         spin_lock_irqsave(&host->lock, flags);
1846
1847         if (host->flags & SDHCI_DEVICE_DEAD)
1848                 is_readonly = 0;
1849         else if (host->ops->get_ro)
1850                 is_readonly = host->ops->get_ro(host);
1851         else
1852                 is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
1853                                 & SDHCI_WRITE_PROTECT);
1854
1855         spin_unlock_irqrestore(&host->lock, flags);
1856
1857         /* This quirk needs to be replaced by a callback-function later */
1858         return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
1859                 !is_readonly : is_readonly;
1860 }
1861
1862 #define SAMPLE_COUNT    5
1863
1864 static int sdhci_get_ro(struct mmc_host *mmc)
1865 {
1866         struct sdhci_host *host = mmc_priv(mmc);
1867         int i, ro_count;
1868
1869         if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
1870                 return sdhci_check_ro(host);
1871
1872         ro_count = 0;
1873         for (i = 0; i < SAMPLE_COUNT; i++) {
1874                 if (sdhci_check_ro(host)) {
1875                         if (++ro_count > SAMPLE_COUNT / 2)
1876                                 return 1;
1877                 }
1878                 msleep(30);
1879         }
1880         return 0;
1881 }
1882
1883 static void sdhci_hw_reset(struct mmc_host *mmc)
1884 {
1885         struct sdhci_host *host = mmc_priv(mmc);
1886
1887         if (host->ops && host->ops->hw_reset)
1888                 host->ops->hw_reset(host);
1889 }
1890
1891 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
1892 {
1893         if (!(host->flags & SDHCI_DEVICE_DEAD)) {
1894                 if (enable)
1895                         host->ier |= SDHCI_INT_CARD_INT;
1896                 else
1897                         host->ier &= ~SDHCI_INT_CARD_INT;
1898
1899                 sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1900                 sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1901                 mmiowb();
1902         }
1903 }
1904
1905 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1906 {
1907         struct sdhci_host *host = mmc_priv(mmc);
1908         unsigned long flags;
1909
1910         if (enable)
1911                 pm_runtime_get_noresume(host->mmc->parent);
1912
1913         spin_lock_irqsave(&host->lock, flags);
1914         if (enable)
1915                 host->flags |= SDHCI_SDIO_IRQ_ENABLED;
1916         else
1917                 host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
1918
1919         sdhci_enable_sdio_irq_nolock(host, enable);
1920         spin_unlock_irqrestore(&host->lock, flags);
1921
1922         if (!enable)
1923                 pm_runtime_put_noidle(host->mmc->parent);
1924 }
1925 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
1926
1927 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
1928                                       struct mmc_ios *ios)
1929 {
1930         struct sdhci_host *host = mmc_priv(mmc);
1931         u16 ctrl;
1932         int ret;
1933
1934         /*
1935          * Signal Voltage Switching is only applicable for Host Controllers
1936          * v3.00 and above.
1937          */
1938         if (host->version < SDHCI_SPEC_300)
1939                 return 0;
1940
1941         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1942
1943         switch (ios->signal_voltage) {
1944         case MMC_SIGNAL_VOLTAGE_330:
1945                 if (!(host->flags & SDHCI_SIGNALING_330))
1946                         return -EINVAL;
1947                 /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
1948                 ctrl &= ~SDHCI_CTRL_VDD_180;
1949                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1950
1951                 if (!IS_ERR(mmc->supply.vqmmc)) {
1952                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1953                         if (ret) {
1954                                 pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
1955                                         mmc_hostname(mmc));
1956                                 return -EIO;
1957                         }
1958                 }
1959                 /* Wait for 5ms */
1960                 usleep_range(5000, 5500);
1961
1962                 /* 3.3V regulator output should be stable within 5 ms */
1963                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1964                 if (!(ctrl & SDHCI_CTRL_VDD_180))
1965                         return 0;
1966
1967                 pr_warn("%s: 3.3V regulator output did not became stable\n",
1968                         mmc_hostname(mmc));
1969
1970                 return -EAGAIN;
1971         case MMC_SIGNAL_VOLTAGE_180:
1972                 if (!(host->flags & SDHCI_SIGNALING_180))
1973                         return -EINVAL;
1974                 if (!IS_ERR(mmc->supply.vqmmc)) {
1975                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1976                         if (ret) {
1977                                 pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
1978                                         mmc_hostname(mmc));
1979                                 return -EIO;
1980                         }
1981                 }
1982
1983                 /*
1984                  * Enable 1.8V Signal Enable in the Host Control2
1985                  * register
1986                  */
1987                 ctrl |= SDHCI_CTRL_VDD_180;
1988                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1989
1990                 /* Some controller need to do more when switching */
1991                 if (host->ops->voltage_switch)
1992                         host->ops->voltage_switch(host);
1993
1994                 /* 1.8V regulator output should be stable within 5 ms */
1995                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1996                 if (ctrl & SDHCI_CTRL_VDD_180)
1997                         return 0;
1998
1999                 pr_warn("%s: 1.8V regulator output did not became stable\n",
2000                         mmc_hostname(mmc));
2001
2002                 return -EAGAIN;
2003         case MMC_SIGNAL_VOLTAGE_120:
2004                 if (!(host->flags & SDHCI_SIGNALING_120))
2005                         return -EINVAL;
2006                 if (!IS_ERR(mmc->supply.vqmmc)) {
2007                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2008                         if (ret) {
2009                                 pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2010                                         mmc_hostname(mmc));
2011                                 return -EIO;
2012                         }
2013                 }
2014                 return 0;
2015         default:
2016                 /* No signal voltage switch required */
2017                 return 0;
2018         }
2019 }
2020 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2021
2022 static int sdhci_card_busy(struct mmc_host *mmc)
2023 {
2024         struct sdhci_host *host = mmc_priv(mmc);
2025         u32 present_state;
2026
2027         /* Check whether DAT[0] is 0 */
2028         present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2029
2030         return !(present_state & SDHCI_DATA_0_LVL_MASK);
2031 }
2032
2033 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2034 {
2035         struct sdhci_host *host = mmc_priv(mmc);
2036         unsigned long flags;
2037
2038         spin_lock_irqsave(&host->lock, flags);
2039         host->flags |= SDHCI_HS400_TUNING;
2040         spin_unlock_irqrestore(&host->lock, flags);
2041
2042         return 0;
2043 }
2044
2045 static void sdhci_start_tuning(struct sdhci_host *host)
2046 {
2047         u16 ctrl;
2048
2049         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2050         ctrl |= SDHCI_CTRL_EXEC_TUNING;
2051         if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2052                 ctrl |= SDHCI_CTRL_TUNED_CLK;
2053         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2054
2055         /*
2056          * As per the Host Controller spec v3.00, tuning command
2057          * generates Buffer Read Ready interrupt, so enable that.
2058          *
2059          * Note: The spec clearly says that when tuning sequence
2060          * is being performed, the controller does not generate
2061          * interrupts other than Buffer Read Ready interrupt. But
2062          * to make sure we don't hit a controller bug, we _only_
2063          * enable Buffer Read Ready interrupt here.
2064          */
2065         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2066         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2067 }
2068
2069 static void sdhci_end_tuning(struct sdhci_host *host)
2070 {
2071         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2072         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2073 }
2074
2075 static void sdhci_reset_tuning(struct sdhci_host *host)
2076 {
2077         u16 ctrl;
2078
2079         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2080         ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2081         ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2082         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2083 }
2084
2085 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2086 {
2087         sdhci_reset_tuning(host);
2088
2089         sdhci_do_reset(host, SDHCI_RESET_CMD);
2090         sdhci_do_reset(host, SDHCI_RESET_DATA);
2091
2092         sdhci_end_tuning(host);
2093
2094         mmc_abort_tuning(host->mmc, opcode);
2095 }
2096
2097 /*
2098  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2099  * tuning command does not have a data payload (or rather the hardware does it
2100  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2101  * interrupt setup is different to other commands and there is no timeout
2102  * interrupt so special handling is needed.
2103  */
2104 static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2105 {
2106         struct mmc_host *mmc = host->mmc;
2107         struct mmc_command cmd = {};
2108         struct mmc_request mrq = {};
2109         unsigned long flags;
2110         u32 b = host->sdma_boundary;
2111
2112         spin_lock_irqsave(&host->lock, flags);
2113
2114         cmd.opcode = opcode;
2115         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2116         cmd.mrq = &mrq;
2117
2118         mrq.cmd = &cmd;
2119         /*
2120          * In response to CMD19, the card sends 64 bytes of tuning
2121          * block to the Host Controller. So we set the block size
2122          * to 64 here.
2123          */
2124         if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2125             mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2126                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2127         else
2128                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2129
2130         /*
2131          * The tuning block is sent by the card to the host controller.
2132          * So we set the TRNS_READ bit in the Transfer Mode register.
2133          * This also takes care of setting DMA Enable and Multi Block
2134          * Select in the same register to 0.
2135          */
2136         sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2137
2138         sdhci_send_command(host, &cmd);
2139
2140         host->cmd = NULL;
2141
2142         sdhci_del_timer(host, &mrq);
2143
2144         host->tuning_done = 0;
2145
2146         mmiowb();
2147         spin_unlock_irqrestore(&host->lock, flags);
2148
2149         /* Wait for Buffer Read Ready interrupt */
2150         wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2151                            msecs_to_jiffies(50));
2152
2153 }
2154
2155 static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2156 {
2157         int i;
2158
2159         /*
2160          * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2161          * of loops reaches 40 times.
2162          */
2163         for (i = 0; i < MAX_TUNING_LOOP; i++) {
2164                 u16 ctrl;
2165
2166                 sdhci_send_tuning(host, opcode);
2167
2168                 if (!host->tuning_done) {
2169                         pr_debug("%s: Tuning timeout, falling back to fixed sampling clock\n",
2170                                  mmc_hostname(host->mmc));
2171                         sdhci_abort_tuning(host, opcode);
2172                         return;
2173                 }
2174
2175                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2176                 if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2177                         if (ctrl & SDHCI_CTRL_TUNED_CLK)
2178                                 return; /* Success! */
2179                         break;
2180                 }
2181
2182                 /* Spec does not require a delay between tuning cycles */
2183                 if (host->tuning_delay > 0)
2184                         mdelay(host->tuning_delay);
2185         }
2186
2187         pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2188                 mmc_hostname(host->mmc));
2189         sdhci_reset_tuning(host);
2190 }
2191
2192 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2193 {
2194         struct sdhci_host *host = mmc_priv(mmc);
2195         int err = 0;
2196         unsigned int tuning_count = 0;
2197         bool hs400_tuning;
2198
2199         hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2200
2201         if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2202                 tuning_count = host->tuning_count;
2203
2204         /*
2205          * The Host Controller needs tuning in case of SDR104 and DDR50
2206          * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2207          * the Capabilities register.
2208          * If the Host Controller supports the HS200 mode then the
2209          * tuning function has to be executed.
2210          */
2211         switch (host->timing) {
2212         /* HS400 tuning is done in HS200 mode */
2213         case MMC_TIMING_MMC_HS400:
2214                 err = -EINVAL;
2215                 goto out;
2216
2217         case MMC_TIMING_MMC_HS200:
2218                 /*
2219                  * Periodic re-tuning for HS400 is not expected to be needed, so
2220                  * disable it here.
2221                  */
2222                 if (hs400_tuning)
2223                         tuning_count = 0;
2224                 break;
2225
2226         case MMC_TIMING_UHS_SDR104:
2227         case MMC_TIMING_UHS_DDR50:
2228                 break;
2229
2230         case MMC_TIMING_UHS_SDR50:
2231                 if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2232                         break;
2233                 /* FALLTHROUGH */
2234
2235         default:
2236                 goto out;
2237         }
2238
2239         if (host->ops->platform_execute_tuning) {
2240                 err = host->ops->platform_execute_tuning(host, opcode);
2241                 goto out;
2242         }
2243
2244         host->mmc->retune_period = tuning_count;
2245
2246         if (host->tuning_delay < 0)
2247                 host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2248
2249         sdhci_start_tuning(host);
2250
2251         __sdhci_execute_tuning(host, opcode);
2252
2253         sdhci_end_tuning(host);
2254 out:
2255         host->flags &= ~SDHCI_HS400_TUNING;
2256
2257         return err;
2258 }
2259 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2260
2261 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2262 {
2263         /* Host Controller v3.00 defines preset value registers */
2264         if (host->version < SDHCI_SPEC_300)
2265                 return;
2266
2267         /*
2268          * We only enable or disable Preset Value if they are not already
2269          * enabled or disabled respectively. Otherwise, we bail out.
2270          */
2271         if (host->preset_enabled != enable) {
2272                 u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2273
2274                 if (enable)
2275                         ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2276                 else
2277                         ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2278
2279                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2280
2281                 if (enable)
2282                         host->flags |= SDHCI_PV_ENABLED;
2283                 else
2284                         host->flags &= ~SDHCI_PV_ENABLED;
2285
2286                 host->preset_enabled = enable;
2287         }
2288 }
2289
2290 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2291                                 int err)
2292 {
2293         struct sdhci_host *host = mmc_priv(mmc);
2294         struct mmc_data *data = mrq->data;
2295
2296         if (data->host_cookie != COOKIE_UNMAPPED)
2297                 dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2298                              mmc_get_dma_dir(data));
2299
2300         data->host_cookie = COOKIE_UNMAPPED;
2301 }
2302
2303 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2304 {
2305         struct sdhci_host *host = mmc_priv(mmc);
2306
2307         mrq->data->host_cookie = COOKIE_UNMAPPED;
2308
2309         /*
2310          * No pre-mapping in the pre hook if we're using the bounce buffer,
2311          * for that we would need two bounce buffers since one buffer is
2312          * in flight when this is getting called.
2313          */
2314         if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2315                 sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2316 }
2317
2318 static inline bool sdhci_has_requests(struct sdhci_host *host)
2319 {
2320         return host->cmd || host->data_cmd;
2321 }
2322
2323 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2324 {
2325         if (host->data_cmd) {
2326                 host->data_cmd->error = err;
2327                 sdhci_finish_mrq(host, host->data_cmd->mrq);
2328         }
2329
2330         if (host->cmd) {
2331                 host->cmd->error = err;
2332                 sdhci_finish_mrq(host, host->cmd->mrq);
2333         }
2334 }
2335
2336 static void sdhci_card_event(struct mmc_host *mmc)
2337 {
2338         struct sdhci_host *host = mmc_priv(mmc);
2339         unsigned long flags;
2340         int present;
2341
2342         /* First check if client has provided their own card event */
2343         if (host->ops->card_event)
2344                 host->ops->card_event(host);
2345
2346         present = mmc->ops->get_cd(mmc);
2347
2348         spin_lock_irqsave(&host->lock, flags);
2349
2350         /* Check sdhci_has_requests() first in case we are runtime suspended */
2351         if (sdhci_has_requests(host) && !present) {
2352                 pr_err("%s: Card removed during transfer!\n",
2353                         mmc_hostname(host->mmc));
2354                 pr_err("%s: Resetting controller.\n",
2355                         mmc_hostname(host->mmc));
2356
2357                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2358                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2359
2360                 sdhci_error_out_mrqs(host, -ENOMEDIUM);
2361         }
2362
2363         spin_unlock_irqrestore(&host->lock, flags);
2364 }
2365
2366 static const struct mmc_host_ops sdhci_ops = {
2367         .request        = sdhci_request,
2368         .post_req       = sdhci_post_req,
2369         .pre_req        = sdhci_pre_req,
2370         .set_ios        = sdhci_set_ios,
2371         .get_cd         = sdhci_get_cd,
2372         .get_ro         = sdhci_get_ro,
2373         .hw_reset       = sdhci_hw_reset,
2374         .enable_sdio_irq = sdhci_enable_sdio_irq,
2375         .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2376         .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2377         .execute_tuning                 = sdhci_execute_tuning,
2378         .card_event                     = sdhci_card_event,
2379         .card_busy      = sdhci_card_busy,
2380 };
2381
2382 /*****************************************************************************\
2383  *                                                                           *
2384  * Tasklets                                                                  *
2385  *                                                                           *
2386 \*****************************************************************************/
2387
2388 static bool sdhci_request_done(struct sdhci_host *host)
2389 {
2390         unsigned long flags;
2391         struct mmc_request *mrq;
2392         int i;
2393
2394         spin_lock_irqsave(&host->lock, flags);
2395
2396         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2397                 mrq = host->mrqs_done[i];
2398                 if (mrq)
2399                         break;
2400         }
2401
2402         if (!mrq) {
2403                 spin_unlock_irqrestore(&host->lock, flags);
2404                 return true;
2405         }
2406
2407         sdhci_del_timer(host, mrq);
2408
2409         /*
2410          * Always unmap the data buffers if they were mapped by
2411          * sdhci_prepare_data() whenever we finish with a request.
2412          * This avoids leaking DMA mappings on error.
2413          */
2414         if (host->flags & SDHCI_REQ_USE_DMA) {
2415                 struct mmc_data *data = mrq->data;
2416
2417                 if (data && data->host_cookie == COOKIE_MAPPED) {
2418                         if (host->bounce_buffer) {
2419                                 /*
2420                                  * On reads, copy the bounced data into the
2421                                  * sglist
2422                                  */
2423                                 if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2424                                         unsigned int length = data->bytes_xfered;
2425
2426                                         if (length > host->bounce_buffer_size) {
2427                                                 pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2428                                                        mmc_hostname(host->mmc),
2429                                                        host->bounce_buffer_size,
2430                                                        data->bytes_xfered);
2431                                                 /* Cap it down and continue */
2432                                                 length = host->bounce_buffer_size;
2433                                         }
2434                                         dma_sync_single_for_cpu(
2435                                                 host->mmc->parent,
2436                                                 host->bounce_addr,
2437                                                 host->bounce_buffer_size,
2438                                                 DMA_FROM_DEVICE);
2439                                         sg_copy_from_buffer(data->sg,
2440                                                 data->sg_len,
2441                                                 host->bounce_buffer,
2442                                                 length);
2443                                 } else {
2444                                         /* No copying, just switch ownership */
2445                                         dma_sync_single_for_cpu(
2446                                                 host->mmc->parent,
2447                                                 host->bounce_addr,
2448                                                 host->bounce_buffer_size,
2449                                                 mmc_get_dma_dir(data));
2450                                 }
2451                         } else {
2452                                 /* Unmap the raw data */
2453                                 dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2454                                              data->sg_len,
2455                                              mmc_get_dma_dir(data));
2456                         }
2457                         data->host_cookie = COOKIE_UNMAPPED;
2458                 }
2459         }
2460
2461         /*
2462          * The controller needs a reset of internal state machines
2463          * upon error conditions.
2464          */
2465         if (sdhci_needs_reset(host, mrq)) {
2466                 /*
2467                  * Do not finish until command and data lines are available for
2468                  * reset. Note there can only be one other mrq, so it cannot
2469                  * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2470                  * would both be null.
2471                  */
2472                 if (host->cmd || host->data_cmd) {
2473                         spin_unlock_irqrestore(&host->lock, flags);
2474                         return true;
2475                 }
2476
2477                 /* Some controllers need this kick or reset won't work here */
2478                 if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2479                         /* This is to force an update */
2480                         host->ops->set_clock(host, host->clock);
2481
2482                 /* Spec says we should do both at the same time, but Ricoh
2483                    controllers do not like that. */
2484                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2485                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2486
2487                 host->pending_reset = false;
2488         }
2489
2490         if (!sdhci_has_requests(host))
2491                 sdhci_led_deactivate(host);
2492
2493         host->mrqs_done[i] = NULL;
2494
2495         mmiowb();
2496         spin_unlock_irqrestore(&host->lock, flags);
2497
2498         mmc_request_done(host->mmc, mrq);
2499
2500         return false;
2501 }
2502
2503 static void sdhci_tasklet_finish(unsigned long param)
2504 {
2505         struct sdhci_host *host = (struct sdhci_host *)param;
2506
2507         while (!sdhci_request_done(host))
2508                 ;
2509 }
2510
2511 static void sdhci_timeout_timer(unsigned long data)
2512 {
2513         struct sdhci_host *host;
2514         unsigned long flags;
2515
2516         host = (struct sdhci_host*)data;
2517
2518         spin_lock_irqsave(&host->lock, flags);
2519
2520         if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2521                 pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2522                        mmc_hostname(host->mmc));
2523                 sdhci_dumpregs(host);
2524
2525                 host->cmd->error = -ETIMEDOUT;
2526                 sdhci_finish_mrq(host, host->cmd->mrq);
2527         }
2528
2529         mmiowb();
2530         spin_unlock_irqrestore(&host->lock, flags);
2531 }
2532
2533 static void sdhci_timeout_data_timer(unsigned long data)
2534 {
2535         struct sdhci_host *host;
2536         unsigned long flags;
2537
2538         host = (struct sdhci_host *)data;
2539
2540         spin_lock_irqsave(&host->lock, flags);
2541
2542         if (host->data || host->data_cmd ||
2543             (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2544                 pr_err("%s: Timeout waiting for hardware interrupt.\n",
2545                        mmc_hostname(host->mmc));
2546                 sdhci_dumpregs(host);
2547
2548                 if (host->data) {
2549                         host->data->error = -ETIMEDOUT;
2550                         sdhci_finish_data(host);
2551                 } else if (host->data_cmd) {
2552                         host->data_cmd->error = -ETIMEDOUT;
2553                         sdhci_finish_mrq(host, host->data_cmd->mrq);
2554                 } else {
2555                         host->cmd->error = -ETIMEDOUT;
2556                         sdhci_finish_mrq(host, host->cmd->mrq);
2557                 }
2558         }
2559
2560         mmiowb();
2561         spin_unlock_irqrestore(&host->lock, flags);
2562 }
2563
2564 /*****************************************************************************\
2565  *                                                                           *
2566  * Interrupt handling                                                        *
2567  *                                                                           *
2568 \*****************************************************************************/
2569
2570 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p)
2571 {
2572         /* Handle auto-CMD12 error */
2573         if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {
2574                 struct mmc_request *mrq = host->data_cmd->mrq;
2575                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2576                 int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2577                                    SDHCI_INT_DATA_TIMEOUT :
2578                                    SDHCI_INT_DATA_CRC;
2579
2580                 /* Treat auto-CMD12 error the same as data error */
2581                 if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {
2582                         *intmask_p |= data_err_bit;
2583                         return;
2584                 }
2585         }
2586
2587         if (!host->cmd) {
2588                 /*
2589                  * SDHCI recovers from errors by resetting the cmd and data
2590                  * circuits.  Until that is done, there very well might be more
2591                  * interrupts, so ignore them in that case.
2592                  */
2593                 if (host->pending_reset)
2594                         return;
2595                 pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2596                        mmc_hostname(host->mmc), (unsigned)intmask);
2597                 sdhci_dumpregs(host);
2598                 return;
2599         }
2600
2601         if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2602                        SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2603                 if (intmask & SDHCI_INT_TIMEOUT)
2604                         host->cmd->error = -ETIMEDOUT;
2605                 else
2606                         host->cmd->error = -EILSEQ;
2607
2608                 /* Treat data command CRC error the same as data CRC error */
2609                 if (host->cmd->data &&
2610                     (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2611                      SDHCI_INT_CRC) {
2612                         host->cmd = NULL;
2613                         *intmask_p |= SDHCI_INT_DATA_CRC;
2614                         return;
2615                 }
2616
2617                 sdhci_finish_mrq(host, host->cmd->mrq);
2618                 return;
2619         }
2620
2621         /* Handle auto-CMD23 error */
2622         if (intmask & SDHCI_INT_AUTO_CMD_ERR) {
2623                 struct mmc_request *mrq = host->cmd->mrq;
2624                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2625                 int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2626                           -ETIMEDOUT :
2627                           -EILSEQ;
2628
2629                 if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
2630                         mrq->sbc->error = err;
2631                         sdhci_finish_mrq(host, mrq);
2632                         return;
2633                 }
2634         }
2635
2636         if (intmask & SDHCI_INT_RESPONSE)
2637                 sdhci_finish_command(host);
2638 }
2639
2640 static void sdhci_adma_show_error(struct sdhci_host *host)
2641 {
2642         void *desc = host->adma_table;
2643         dma_addr_t dma = host->adma_addr;
2644
2645         sdhci_dumpregs(host);
2646
2647         while (true) {
2648                 struct sdhci_adma2_64_desc *dma_desc = desc;
2649
2650                 if (host->flags & SDHCI_USE_64_BIT_DMA)
2651                         SDHCI_DUMP("%08llx: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2652                             (unsigned long long)dma,
2653                             le32_to_cpu(dma_desc->addr_hi),
2654                             le32_to_cpu(dma_desc->addr_lo),
2655                             le16_to_cpu(dma_desc->len),
2656                             le16_to_cpu(dma_desc->cmd));
2657                 else
2658                         SDHCI_DUMP("%08llx: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2659                             (unsigned long long)dma,
2660                             le32_to_cpu(dma_desc->addr_lo),
2661                             le16_to_cpu(dma_desc->len),
2662                             le16_to_cpu(dma_desc->cmd));
2663
2664                 desc += host->desc_sz;
2665                 dma += host->desc_sz;
2666
2667                 if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2668                         break;
2669         }
2670 }
2671
2672 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2673 {
2674         u32 command;
2675
2676         /* CMD19 generates _only_ Buffer Read Ready interrupt */
2677         if (intmask & SDHCI_INT_DATA_AVAIL) {
2678                 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2679                 if (command == MMC_SEND_TUNING_BLOCK ||
2680                     command == MMC_SEND_TUNING_BLOCK_HS200) {
2681                         host->tuning_done = 1;
2682                         wake_up(&host->buf_ready_int);
2683                         return;
2684                 }
2685         }
2686
2687         if (!host->data) {
2688                 struct mmc_command *data_cmd = host->data_cmd;
2689
2690                 /*
2691                  * The "data complete" interrupt is also used to
2692                  * indicate that a busy state has ended. See comment
2693                  * above in sdhci_cmd_irq().
2694                  */
2695                 if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2696                         if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2697                                 host->data_cmd = NULL;
2698                                 data_cmd->error = -ETIMEDOUT;
2699                                 sdhci_finish_mrq(host, data_cmd->mrq);
2700                                 return;
2701                         }
2702                         if (intmask & SDHCI_INT_DATA_END) {
2703                                 host->data_cmd = NULL;
2704                                 /*
2705                                  * Some cards handle busy-end interrupt
2706                                  * before the command completed, so make
2707                                  * sure we do things in the proper order.
2708                                  */
2709                                 if (host->cmd == data_cmd)
2710                                         return;
2711
2712                                 sdhci_finish_mrq(host, data_cmd->mrq);
2713                                 return;
2714                         }
2715                 }
2716
2717                 /*
2718                  * SDHCI recovers from errors by resetting the cmd and data
2719                  * circuits. Until that is done, there very well might be more
2720                  * interrupts, so ignore them in that case.
2721                  */
2722                 if (host->pending_reset)
2723                         return;
2724
2725                 pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2726                        mmc_hostname(host->mmc), (unsigned)intmask);
2727                 sdhci_dumpregs(host);
2728
2729                 return;
2730         }
2731
2732         if (intmask & SDHCI_INT_DATA_TIMEOUT)
2733                 host->data->error = -ETIMEDOUT;
2734         else if (intmask & SDHCI_INT_DATA_END_BIT)
2735                 host->data->error = -EILSEQ;
2736         else if ((intmask & SDHCI_INT_DATA_CRC) &&
2737                 SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2738                         != MMC_BUS_TEST_R)
2739                 host->data->error = -EILSEQ;
2740         else if (intmask & SDHCI_INT_ADMA_ERROR) {
2741                 pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc),
2742                        intmask);
2743                 sdhci_adma_show_error(host);
2744                 host->data->error = -EIO;
2745                 if (host->ops->adma_workaround)
2746                         host->ops->adma_workaround(host, intmask);
2747         }
2748
2749         if (host->data->error)
2750                 sdhci_finish_data(host);
2751         else {
2752                 if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2753                         sdhci_transfer_pio(host);
2754
2755                 /*
2756                  * We currently don't do anything fancy with DMA
2757                  * boundaries, but as we can't disable the feature
2758                  * we need to at least restart the transfer.
2759                  *
2760                  * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2761                  * should return a valid address to continue from, but as
2762                  * some controllers are faulty, don't trust them.
2763                  */
2764                 if (intmask & SDHCI_INT_DMA_END) {
2765                         u32 dmastart, dmanow;
2766
2767                         dmastart = sdhci_sdma_address(host);
2768                         dmanow = dmastart + host->data->bytes_xfered;
2769                         /*
2770                          * Force update to the next DMA block boundary.
2771                          */
2772                         dmanow = (dmanow &
2773                                 ~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2774                                 SDHCI_DEFAULT_BOUNDARY_SIZE;
2775                         host->data->bytes_xfered = dmanow - dmastart;
2776                         DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n",
2777                             dmastart, host->data->bytes_xfered, dmanow);
2778                         sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS);
2779                 }
2780
2781                 if (intmask & SDHCI_INT_DATA_END) {
2782                         if (host->cmd == host->data_cmd) {
2783                                 /*
2784                                  * Data managed to finish before the
2785                                  * command completed. Make sure we do
2786                                  * things in the proper order.
2787                                  */
2788                                 host->data_early = 1;
2789                         } else {
2790                                 sdhci_finish_data(host);
2791                         }
2792                 }
2793         }
2794 }
2795
2796 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2797 {
2798         irqreturn_t result = IRQ_NONE;
2799         struct sdhci_host *host = dev_id;
2800         u32 intmask, mask, unexpected = 0;
2801         int max_loops = 16;
2802
2803         spin_lock(&host->lock);
2804
2805         if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
2806                 spin_unlock(&host->lock);
2807                 return IRQ_NONE;
2808         }
2809
2810         intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2811         if (!intmask || intmask == 0xffffffff) {
2812                 result = IRQ_NONE;
2813                 goto out;
2814         }
2815
2816         do {
2817                 DBG("IRQ status 0x%08x\n", intmask);
2818
2819                 if (host->ops->irq) {
2820                         intmask = host->ops->irq(host, intmask);
2821                         if (!intmask)
2822                                 goto cont;
2823                 }
2824
2825                 /* Clear selected interrupts. */
2826                 mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2827                                   SDHCI_INT_BUS_POWER);
2828                 sdhci_writel(host, mask, SDHCI_INT_STATUS);
2829
2830                 if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2831                         u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
2832                                       SDHCI_CARD_PRESENT;
2833
2834                         /*
2835                          * There is a observation on i.mx esdhc.  INSERT
2836                          * bit will be immediately set again when it gets
2837                          * cleared, if a card is inserted.  We have to mask
2838                          * the irq to prevent interrupt storm which will
2839                          * freeze the system.  And the REMOVE gets the
2840                          * same situation.
2841                          *
2842                          * More testing are needed here to ensure it works
2843                          * for other platforms though.
2844                          */
2845                         host->ier &= ~(SDHCI_INT_CARD_INSERT |
2846                                        SDHCI_INT_CARD_REMOVE);
2847                         host->ier |= present ? SDHCI_INT_CARD_REMOVE :
2848                                                SDHCI_INT_CARD_INSERT;
2849                         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2850                         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2851
2852                         sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
2853                                      SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
2854
2855                         host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
2856                                                        SDHCI_INT_CARD_REMOVE);
2857                         result = IRQ_WAKE_THREAD;
2858                 }
2859
2860                 if (intmask & SDHCI_INT_CMD_MASK)
2861                         sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask);
2862
2863                 if (intmask & SDHCI_INT_DATA_MASK)
2864                         sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
2865
2866                 if (intmask & SDHCI_INT_BUS_POWER)
2867                         pr_err("%s: Card is consuming too much power!\n",
2868                                 mmc_hostname(host->mmc));
2869
2870                 if (intmask & SDHCI_INT_RETUNE)
2871                         mmc_retune_needed(host->mmc);
2872
2873                 if ((intmask & SDHCI_INT_CARD_INT) &&
2874                     (host->ier & SDHCI_INT_CARD_INT)) {
2875                         sdhci_enable_sdio_irq_nolock(host, false);
2876                         host->thread_isr |= SDHCI_INT_CARD_INT;
2877                         result = IRQ_WAKE_THREAD;
2878                 }
2879
2880                 intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2881                              SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2882                              SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
2883                              SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
2884
2885                 if (intmask) {
2886                         unexpected |= intmask;
2887                         sdhci_writel(host, intmask, SDHCI_INT_STATUS);
2888                 }
2889 cont:
2890                 if (result == IRQ_NONE)
2891                         result = IRQ_HANDLED;
2892
2893                 intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2894         } while (intmask && --max_loops);
2895 out:
2896         spin_unlock(&host->lock);
2897
2898         if (unexpected) {
2899                 pr_err("%s: Unexpected interrupt 0x%08x.\n",
2900                            mmc_hostname(host->mmc), unexpected);
2901                 sdhci_dumpregs(host);
2902         }
2903
2904         return result;
2905 }
2906
2907 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
2908 {
2909         struct sdhci_host *host = dev_id;
2910         unsigned long flags;
2911         u32 isr;
2912
2913         spin_lock_irqsave(&host->lock, flags);
2914         isr = host->thread_isr;
2915         host->thread_isr = 0;
2916         spin_unlock_irqrestore(&host->lock, flags);
2917
2918         if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2919                 struct mmc_host *mmc = host->mmc;
2920
2921                 mmc->ops->card_event(mmc);
2922                 mmc_detect_change(mmc, msecs_to_jiffies(200));
2923         }
2924
2925         if (isr & SDHCI_INT_CARD_INT) {
2926                 sdio_run_irqs(host->mmc);
2927
2928                 spin_lock_irqsave(&host->lock, flags);
2929                 if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
2930                         sdhci_enable_sdio_irq_nolock(host, true);
2931                 spin_unlock_irqrestore(&host->lock, flags);
2932         }
2933
2934         return isr ? IRQ_HANDLED : IRQ_NONE;
2935 }
2936
2937 /*****************************************************************************\
2938  *                                                                           *
2939  * Suspend/resume                                                            *
2940  *                                                                           *
2941 \*****************************************************************************/
2942
2943 #ifdef CONFIG_PM
2944 /*
2945  * To enable wakeup events, the corresponding events have to be enabled in
2946  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
2947  * Table' in the SD Host Controller Standard Specification.
2948  * It is useless to restore SDHCI_INT_ENABLE state in
2949  * sdhci_disable_irq_wakeups() since it will be set by
2950  * sdhci_enable_card_detection() or sdhci_init().
2951  */
2952 void sdhci_enable_irq_wakeups(struct sdhci_host *host)
2953 {
2954         u8 val;
2955         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
2956                         | SDHCI_WAKE_ON_INT;
2957         u32 irq_val = SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2958                       SDHCI_INT_CARD_INT;
2959
2960         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2961         val |= mask ;
2962         /* Avoid fake wake up */
2963         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) {
2964                 val &= ~(SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE);
2965                 irq_val &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
2966         }
2967         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2968         sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
2969 }
2970 EXPORT_SYMBOL_GPL(sdhci_enable_irq_wakeups);
2971
2972 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
2973 {
2974         u8 val;
2975         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
2976                         | SDHCI_WAKE_ON_INT;
2977
2978         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2979         val &= ~mask;
2980         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2981 }
2982
2983 int sdhci_suspend_host(struct sdhci_host *host)
2984 {
2985         sdhci_disable_card_detection(host);
2986
2987         mmc_retune_timer_stop(host->mmc);
2988
2989         if (!device_may_wakeup(mmc_dev(host->mmc))) {
2990                 host->ier = 0;
2991                 sdhci_writel(host, 0, SDHCI_INT_ENABLE);
2992                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
2993                 free_irq(host->irq, host);
2994         } else {
2995                 sdhci_enable_irq_wakeups(host);
2996                 enable_irq_wake(host->irq);
2997         }
2998         return 0;
2999 }
3000
3001 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3002
3003 int sdhci_resume_host(struct sdhci_host *host)
3004 {
3005         struct mmc_host *mmc = host->mmc;
3006         int ret = 0;
3007
3008         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3009                 if (host->ops->enable_dma)
3010                         host->ops->enable_dma(host);
3011         }
3012
3013         if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3014             (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3015                 /* Card keeps power but host controller does not */
3016                 sdhci_init(host, 0);
3017                 host->pwr = 0;
3018                 host->clock = 0;
3019                 mmc->ops->set_ios(mmc, &mmc->ios);
3020         } else {
3021                 sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3022                 mmiowb();
3023         }
3024
3025         if (!device_may_wakeup(mmc_dev(host->mmc))) {
3026                 ret = request_threaded_irq(host->irq, sdhci_irq,
3027                                            sdhci_thread_irq, IRQF_SHARED,
3028                                            mmc_hostname(host->mmc), host);
3029                 if (ret)
3030                         return ret;
3031         } else {
3032                 sdhci_disable_irq_wakeups(host);
3033                 disable_irq_wake(host->irq);
3034         }
3035
3036         sdhci_enable_card_detection(host);
3037
3038         return ret;
3039 }
3040
3041 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3042
3043 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3044 {
3045         unsigned long flags;
3046
3047         mmc_retune_timer_stop(host->mmc);
3048
3049         spin_lock_irqsave(&host->lock, flags);
3050         host->ier &= SDHCI_INT_CARD_INT;
3051         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3052         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3053         spin_unlock_irqrestore(&host->lock, flags);
3054
3055         synchronize_hardirq(host->irq);
3056
3057         spin_lock_irqsave(&host->lock, flags);
3058         host->runtime_suspended = true;
3059         spin_unlock_irqrestore(&host->lock, flags);
3060
3061         return 0;
3062 }
3063 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3064
3065 int sdhci_runtime_resume_host(struct sdhci_host *host)
3066 {
3067         struct mmc_host *mmc = host->mmc;
3068         unsigned long flags;
3069         int host_flags = host->flags;
3070
3071         if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3072                 if (host->ops->enable_dma)
3073                         host->ops->enable_dma(host);
3074         }
3075
3076         sdhci_init(host, 0);
3077
3078         if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3079             mmc->ios.power_mode != MMC_POWER_OFF) {
3080                 /* Force clock and power re-program */
3081                 host->pwr = 0;
3082                 host->clock = 0;
3083                 mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3084                 mmc->ops->set_ios(mmc, &mmc->ios);
3085
3086                 if ((host_flags & SDHCI_PV_ENABLED) &&
3087                     !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3088                         spin_lock_irqsave(&host->lock, flags);
3089                         sdhci_enable_preset_value(host, true);
3090                         spin_unlock_irqrestore(&host->lock, flags);
3091                 }
3092
3093                 if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3094                     mmc->ops->hs400_enhanced_strobe)
3095                         mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3096         }
3097
3098         spin_lock_irqsave(&host->lock, flags);
3099
3100         host->runtime_suspended = false;
3101
3102         /* Enable SDIO IRQ */
3103         if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3104                 sdhci_enable_sdio_irq_nolock(host, true);
3105
3106         /* Enable Card Detection */
3107         sdhci_enable_card_detection(host);
3108
3109         spin_unlock_irqrestore(&host->lock, flags);
3110
3111         return 0;
3112 }
3113 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3114
3115 #endif /* CONFIG_PM */
3116
3117 /*****************************************************************************\
3118  *                                                                           *
3119  * Command Queue Engine (CQE) helpers                                        *
3120  *                                                                           *
3121 \*****************************************************************************/
3122
3123 void sdhci_cqe_enable(struct mmc_host *mmc)
3124 {
3125         struct sdhci_host *host = mmc_priv(mmc);
3126         unsigned long flags;
3127         u8 ctrl;
3128
3129         spin_lock_irqsave(&host->lock, flags);
3130
3131         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3132         ctrl &= ~SDHCI_CTRL_DMA_MASK;
3133         if (host->flags & SDHCI_USE_64_BIT_DMA)
3134                 ctrl |= SDHCI_CTRL_ADMA64;
3135         else
3136                 ctrl |= SDHCI_CTRL_ADMA32;
3137         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3138
3139         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3140                      SDHCI_BLOCK_SIZE);
3141
3142         /* Set maximum timeout */
3143         sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
3144
3145         host->ier = host->cqe_ier;
3146
3147         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3148         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3149
3150         host->cqe_on = true;
3151
3152         pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3153                  mmc_hostname(mmc), host->ier,
3154                  sdhci_readl(host, SDHCI_INT_STATUS));
3155
3156         mmiowb();
3157         spin_unlock_irqrestore(&host->lock, flags);
3158 }
3159 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3160
3161 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3162 {
3163         struct sdhci_host *host = mmc_priv(mmc);
3164         unsigned long flags;
3165
3166         spin_lock_irqsave(&host->lock, flags);
3167
3168         sdhci_set_default_irqs(host);
3169
3170         host->cqe_on = false;
3171
3172         if (recovery) {
3173                 sdhci_do_reset(host, SDHCI_RESET_CMD);
3174                 sdhci_do_reset(host, SDHCI_RESET_DATA);
3175         }
3176
3177         pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3178                  mmc_hostname(mmc), host->ier,
3179                  sdhci_readl(host, SDHCI_INT_STATUS));
3180
3181         mmiowb();
3182         spin_unlock_irqrestore(&host->lock, flags);
3183 }
3184 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3185
3186 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3187                    int *data_error)
3188 {
3189         u32 mask;
3190
3191         if (!host->cqe_on)
3192                 return false;
3193
3194         if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3195                 *cmd_error = -EILSEQ;
3196         else if (intmask & SDHCI_INT_TIMEOUT)
3197                 *cmd_error = -ETIMEDOUT;
3198         else
3199                 *cmd_error = 0;
3200
3201         if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3202                 *data_error = -EILSEQ;
3203         else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3204                 *data_error = -ETIMEDOUT;
3205         else if (intmask & SDHCI_INT_ADMA_ERROR)
3206                 *data_error = -EIO;
3207         else
3208                 *data_error = 0;
3209
3210         /* Clear selected interrupts. */
3211         mask = intmask & host->cqe_ier;
3212         sdhci_writel(host, mask, SDHCI_INT_STATUS);
3213
3214         if (intmask & SDHCI_INT_BUS_POWER)
3215                 pr_err("%s: Card is consuming too much power!\n",
3216                        mmc_hostname(host->mmc));
3217
3218         intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3219         if (intmask) {
3220                 sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3221                 pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3222                        mmc_hostname(host->mmc), intmask);
3223                 sdhci_dumpregs(host);
3224         }
3225
3226         return true;
3227 }
3228 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3229
3230 /*****************************************************************************\
3231  *                                                                           *
3232  * Device allocation/registration                                            *
3233  *                                                                           *
3234 \*****************************************************************************/
3235
3236 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3237         size_t priv_size)
3238 {
3239         struct mmc_host *mmc;
3240         struct sdhci_host *host;
3241
3242         WARN_ON(dev == NULL);
3243
3244         mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3245         if (!mmc)
3246                 return ERR_PTR(-ENOMEM);
3247
3248         host = mmc_priv(mmc);
3249         host->mmc = mmc;
3250         host->mmc_host_ops = sdhci_ops;
3251         mmc->ops = &host->mmc_host_ops;
3252
3253         host->flags = SDHCI_SIGNALING_330;
3254
3255         host->cqe_ier     = SDHCI_CQE_INT_MASK;
3256         host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3257
3258         host->tuning_delay = -1;
3259
3260         host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3261
3262         return host;
3263 }
3264
3265 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3266
3267 static int sdhci_set_dma_mask(struct sdhci_host *host)
3268 {
3269         struct mmc_host *mmc = host->mmc;
3270         struct device *dev = mmc_dev(mmc);
3271         int ret = -EINVAL;
3272
3273         if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3274                 host->flags &= ~SDHCI_USE_64_BIT_DMA;
3275
3276         /* Try 64-bit mask if hardware is capable  of it */
3277         if (host->flags & SDHCI_USE_64_BIT_DMA) {
3278                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3279                 if (ret) {
3280                         pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3281                                 mmc_hostname(mmc));
3282                         host->flags &= ~SDHCI_USE_64_BIT_DMA;
3283                 }
3284         }
3285
3286         /* 32-bit mask as default & fallback */
3287         if (ret) {
3288                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3289                 if (ret)
3290                         pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3291                                 mmc_hostname(mmc));
3292         }
3293
3294         return ret;
3295 }
3296
3297 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3298 {
3299         u16 v;
3300         u64 dt_caps_mask = 0;
3301         u64 dt_caps = 0;
3302
3303         if (host->read_caps)
3304                 return;
3305
3306         host->read_caps = true;
3307
3308         if (debug_quirks)
3309                 host->quirks = debug_quirks;
3310
3311         if (debug_quirks2)
3312                 host->quirks2 = debug_quirks2;
3313
3314         sdhci_do_reset(host, SDHCI_RESET_ALL);
3315
3316         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3317                              "sdhci-caps-mask", &dt_caps_mask);
3318         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3319                              "sdhci-caps", &dt_caps);
3320
3321         v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3322         host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3323
3324         if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3325                 return;
3326
3327         if (caps) {
3328                 host->caps = *caps;
3329         } else {
3330                 host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3331                 host->caps &= ~lower_32_bits(dt_caps_mask);
3332                 host->caps |= lower_32_bits(dt_caps);
3333         }
3334
3335         if (host->version < SDHCI_SPEC_300)
3336                 return;
3337
3338         if (caps1) {
3339                 host->caps1 = *caps1;
3340         } else {
3341                 host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3342                 host->caps1 &= ~upper_32_bits(dt_caps_mask);
3343                 host->caps1 |= upper_32_bits(dt_caps);
3344         }
3345 }
3346 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3347
3348 static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3349 {
3350         struct mmc_host *mmc = host->mmc;
3351         unsigned int max_blocks;
3352         unsigned int bounce_size;
3353         int ret;
3354
3355         /*
3356          * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3357          * has diminishing returns, this is probably because SD/MMC
3358          * cards are usually optimized to handle this size of requests.
3359          */
3360         bounce_size = SZ_64K;
3361         /*
3362          * Adjust downwards to maximum request size if this is less
3363          * than our segment size, else hammer down the maximum
3364          * request size to the maximum buffer size.
3365          */
3366         if (mmc->max_req_size < bounce_size)
3367                 bounce_size = mmc->max_req_size;
3368         max_blocks = bounce_size / 512;
3369
3370         /*
3371          * When we just support one segment, we can get significant
3372          * speedups by the help of a bounce buffer to group scattered
3373          * reads/writes together.
3374          */
3375         host->bounce_buffer = devm_kmalloc(mmc->parent,
3376                                            bounce_size,
3377                                            GFP_KERNEL);
3378         if (!host->bounce_buffer) {
3379                 pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3380                        mmc_hostname(mmc),
3381                        bounce_size);
3382                 /*
3383                  * Exiting with zero here makes sure we proceed with
3384                  * mmc->max_segs == 1.
3385                  */
3386                 return 0;
3387         }
3388
3389         host->bounce_addr = dma_map_single(mmc->parent,
3390                                            host->bounce_buffer,
3391                                            bounce_size,
3392                                            DMA_BIDIRECTIONAL);
3393         ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3394         if (ret)
3395                 /* Again fall back to max_segs == 1 */
3396                 return 0;
3397         host->bounce_buffer_size = bounce_size;
3398
3399         /* Lie about this since we're bouncing */
3400         mmc->max_segs = max_blocks;
3401         mmc->max_seg_size = bounce_size;
3402         mmc->max_req_size = bounce_size;
3403
3404         pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3405                 mmc_hostname(mmc), max_blocks, bounce_size);
3406
3407         return 0;
3408 }
3409
3410 int sdhci_setup_host(struct sdhci_host *host)
3411 {
3412         struct mmc_host *mmc;
3413         u32 max_current_caps;
3414         unsigned int ocr_avail;
3415         unsigned int override_timeout_clk;
3416         u32 max_clk;
3417         int ret;
3418
3419         WARN_ON(host == NULL);
3420         if (host == NULL)
3421                 return -EINVAL;
3422
3423         mmc = host->mmc;
3424
3425         /*
3426          * If there are external regulators, get them. Note this must be done
3427          * early before resetting the host and reading the capabilities so that
3428          * the host can take the appropriate action if regulators are not
3429          * available.
3430          */
3431         ret = mmc_regulator_get_supply(mmc);
3432         if (ret == -EPROBE_DEFER)
3433                 return ret;
3434
3435         DBG("Version:   0x%08x | Present:  0x%08x\n",
3436             sdhci_readw(host, SDHCI_HOST_VERSION),
3437             sdhci_readl(host, SDHCI_PRESENT_STATE));
3438         DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3439             sdhci_readl(host, SDHCI_CAPABILITIES),
3440             sdhci_readl(host, SDHCI_CAPABILITIES_1));
3441
3442         sdhci_read_caps(host);
3443
3444         override_timeout_clk = host->timeout_clk;
3445
3446         if (host->version > SDHCI_SPEC_300) {
3447                 pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3448                        mmc_hostname(mmc), host->version);
3449         }
3450
3451         if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3452                 host->flags |= SDHCI_USE_SDMA;
3453         else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3454                 DBG("Controller doesn't have SDMA capability\n");
3455         else
3456                 host->flags |= SDHCI_USE_SDMA;
3457
3458         if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3459                 (host->flags & SDHCI_USE_SDMA)) {
3460                 DBG("Disabling DMA as it is marked broken\n");
3461                 host->flags &= ~SDHCI_USE_SDMA;
3462         }
3463
3464         if ((host->version >= SDHCI_SPEC_200) &&
3465                 (host->caps & SDHCI_CAN_DO_ADMA2))
3466                 host->flags |= SDHCI_USE_ADMA;
3467
3468         if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3469                 (host->flags & SDHCI_USE_ADMA)) {
3470                 DBG("Disabling ADMA as it is marked broken\n");
3471                 host->flags &= ~SDHCI_USE_ADMA;
3472         }
3473
3474         /*
3475          * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3476          * and *must* do 64-bit DMA.  A driver has the opportunity to change
3477          * that during the first call to ->enable_dma().  Similarly
3478          * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3479          * implement.
3480          */
3481         if (host->caps & SDHCI_CAN_64BIT)
3482                 host->flags |= SDHCI_USE_64_BIT_DMA;
3483
3484         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3485                 ret = sdhci_set_dma_mask(host);
3486
3487                 if (!ret && host->ops->enable_dma)
3488                         ret = host->ops->enable_dma(host);
3489
3490                 if (ret) {
3491                         pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3492                                 mmc_hostname(mmc));
3493                         host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3494
3495                         ret = 0;
3496                 }
3497         }
3498
3499         /* SDMA does not support 64-bit DMA */
3500         if (host->flags & SDHCI_USE_64_BIT_DMA)
3501                 host->flags &= ~SDHCI_USE_SDMA;
3502
3503         if (host->flags & SDHCI_USE_ADMA) {
3504                 dma_addr_t dma;
3505                 void *buf;
3506
3507                 /*
3508                  * The DMA descriptor table size is calculated as the maximum
3509                  * number of segments times 2, to allow for an alignment
3510                  * descriptor for each segment, plus 1 for a nop end descriptor,
3511                  * all multipled by the descriptor size.
3512                  */
3513                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
3514                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3515                                               SDHCI_ADMA2_64_DESC_SZ;
3516                         host->desc_sz = SDHCI_ADMA2_64_DESC_SZ;
3517                 } else {
3518                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3519                                               SDHCI_ADMA2_32_DESC_SZ;
3520                         host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3521                 }
3522
3523                 host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3524                 buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz +
3525                                          host->adma_table_sz, &dma, GFP_KERNEL);
3526                 if (!buf) {
3527                         pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3528                                 mmc_hostname(mmc));
3529                         host->flags &= ~SDHCI_USE_ADMA;
3530                 } else if ((dma + host->align_buffer_sz) &
3531                            (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3532                         pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3533                                 mmc_hostname(mmc));
3534                         host->flags &= ~SDHCI_USE_ADMA;
3535                         dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3536                                           host->adma_table_sz, buf, dma);
3537                 } else {
3538                         host->align_buffer = buf;
3539                         host->align_addr = dma;
3540
3541                         host->adma_table = buf + host->align_buffer_sz;
3542                         host->adma_addr = dma + host->align_buffer_sz;
3543                 }
3544         }
3545
3546         /*
3547          * If we use DMA, then it's up to the caller to set the DMA
3548          * mask, but PIO does not need the hw shim so we set a new
3549          * mask here in that case.
3550          */
3551         if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3552                 host->dma_mask = DMA_BIT_MASK(64);
3553                 mmc_dev(mmc)->dma_mask = &host->dma_mask;
3554         }
3555
3556         if (host->version >= SDHCI_SPEC_300)
3557                 host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3558                         >> SDHCI_CLOCK_BASE_SHIFT;
3559         else
3560                 host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3561                         >> SDHCI_CLOCK_BASE_SHIFT;
3562
3563         host->max_clk *= 1000000;
3564         if (host->max_clk == 0 || host->quirks &
3565                         SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3566                 if (!host->ops->get_max_clock) {
3567                         pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3568                                mmc_hostname(mmc));
3569                         ret = -ENODEV;
3570                         goto undma;
3571                 }
3572                 host->max_clk = host->ops->get_max_clock(host);
3573         }
3574
3575         /*
3576          * In case of Host Controller v3.00, find out whether clock
3577          * multiplier is supported.
3578          */
3579         host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3580                         SDHCI_CLOCK_MUL_SHIFT;
3581
3582         /*
3583          * In case the value in Clock Multiplier is 0, then programmable
3584          * clock mode is not supported, otherwise the actual clock
3585          * multiplier is one more than the value of Clock Multiplier
3586          * in the Capabilities Register.
3587          */
3588         if (host->clk_mul)
3589                 host->clk_mul += 1;
3590
3591         /*
3592          * Set host parameters.
3593          */
3594         max_clk = host->max_clk;
3595
3596         if (host->ops->get_min_clock)
3597                 mmc->f_min = host->ops->get_min_clock(host);
3598         else if (host->version >= SDHCI_SPEC_300) {
3599                 if (host->clk_mul)
3600                         max_clk = host->max_clk * host->clk_mul;
3601                 /*
3602                  * Divided Clock Mode minimum clock rate is always less than
3603                  * Programmable Clock Mode minimum clock rate.
3604                  */
3605                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3606         } else
3607                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3608
3609         if (!mmc->f_max || mmc->f_max > max_clk)
3610                 mmc->f_max = max_clk;
3611
3612         if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3613                 host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3614                                         SDHCI_TIMEOUT_CLK_SHIFT;
3615
3616                 if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3617                         host->timeout_clk *= 1000;
3618
3619                 if (host->timeout_clk == 0) {
3620                         if (!host->ops->get_timeout_clock) {
3621                                 pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3622                                         mmc_hostname(mmc));
3623                                 ret = -ENODEV;
3624                                 goto undma;
3625                         }
3626
3627                         host->timeout_clk =
3628                                 DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3629                                              1000);
3630                 }
3631
3632                 if (override_timeout_clk)
3633                         host->timeout_clk = override_timeout_clk;
3634
3635                 mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3636                         host->ops->get_max_timeout_count(host) : 1 << 27;
3637                 mmc->max_busy_timeout /= host->timeout_clk;
3638         }
3639
3640         mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3641         mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3642
3643         if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3644                 host->flags |= SDHCI_AUTO_CMD12;
3645
3646         /* Auto-CMD23 stuff only works in ADMA or PIO. */
3647         if ((host->version >= SDHCI_SPEC_300) &&
3648             ((host->flags & SDHCI_USE_ADMA) ||
3649              !(host->flags & SDHCI_USE_SDMA)) &&
3650              !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3651                 host->flags |= SDHCI_AUTO_CMD23;
3652                 DBG("Auto-CMD23 available\n");
3653         } else {
3654                 DBG("Auto-CMD23 unavailable\n");
3655         }
3656
3657         /*
3658          * A controller may support 8-bit width, but the board itself
3659          * might not have the pins brought out.  Boards that support
3660          * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3661          * their platform code before calling sdhci_add_host(), and we
3662          * won't assume 8-bit width for hosts without that CAP.
3663          */
3664         if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3665                 mmc->caps |= MMC_CAP_4_BIT_DATA;
3666
3667         if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3668                 mmc->caps &= ~MMC_CAP_CMD23;
3669
3670         if (host->caps & SDHCI_CAN_DO_HISPD)
3671                 mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3672
3673         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3674             mmc_card_is_removable(mmc) &&
3675             mmc_gpio_get_cd(host->mmc) < 0)
3676                 mmc->caps |= MMC_CAP_NEEDS_POLL;
3677
3678         if (!IS_ERR(mmc->supply.vqmmc)) {
3679                 ret = regulator_enable(mmc->supply.vqmmc);
3680
3681                 /* If vqmmc provides no 1.8V signalling, then there's no UHS */
3682                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3683                                                     1950000))
3684                         host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3685                                          SDHCI_SUPPORT_SDR50 |
3686                                          SDHCI_SUPPORT_DDR50);
3687
3688                 /* In eMMC case vqmmc might be a fixed 1.8V regulator */
3689                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,
3690                                                     3600000))
3691                         host->flags &= ~SDHCI_SIGNALING_330;
3692
3693                 if (ret) {
3694                         pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3695                                 mmc_hostname(mmc), ret);
3696                         mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3697                 }
3698         }
3699
3700         if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3701                 host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3702                                  SDHCI_SUPPORT_DDR50);
3703         }
3704
3705         /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3706         if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3707                            SDHCI_SUPPORT_DDR50))
3708                 mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3709
3710         /* SDR104 supports also implies SDR50 support */
3711         if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3712                 mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3713                 /* SD3.0: SDR104 is supported so (for eMMC) the caps2
3714                  * field can be promoted to support HS200.
3715                  */
3716                 if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3717                         mmc->caps2 |= MMC_CAP2_HS200;
3718         } else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3719                 mmc->caps |= MMC_CAP_UHS_SDR50;
3720         }
3721
3722         if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3723             (host->caps1 & SDHCI_SUPPORT_HS400))
3724                 mmc->caps2 |= MMC_CAP2_HS400;
3725
3726         if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3727             (IS_ERR(mmc->supply.vqmmc) ||
3728              !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3729                                              1300000)))
3730                 mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3731
3732         if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3733             !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3734                 mmc->caps |= MMC_CAP_UHS_DDR50;
3735
3736         /* Does the host need tuning for SDR50? */
3737         if (host->caps1 & SDHCI_USE_SDR50_TUNING)
3738                 host->flags |= SDHCI_SDR50_NEEDS_TUNING;
3739
3740         /* Driver Type(s) (A, C, D) supported by the host */
3741         if (host->caps1 & SDHCI_DRIVER_TYPE_A)
3742                 mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
3743         if (host->caps1 & SDHCI_DRIVER_TYPE_C)
3744                 mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
3745         if (host->caps1 & SDHCI_DRIVER_TYPE_D)
3746                 mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
3747
3748         /* Initial value for re-tuning timer count */
3749         host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
3750                              SDHCI_RETUNING_TIMER_COUNT_SHIFT;
3751
3752         /*
3753          * In case Re-tuning Timer is not disabled, the actual value of
3754          * re-tuning timer will be 2 ^ (n - 1).
3755          */
3756         if (host->tuning_count)
3757                 host->tuning_count = 1 << (host->tuning_count - 1);
3758
3759         /* Re-tuning mode supported by the Host Controller */
3760         host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
3761                              SDHCI_RETUNING_MODE_SHIFT;
3762
3763         ocr_avail = 0;
3764
3765         /*
3766          * According to SD Host Controller spec v3.00, if the Host System
3767          * can afford more than 150mA, Host Driver should set XPC to 1. Also
3768          * the value is meaningful only if Voltage Support in the Capabilities
3769          * register is set. The actual current value is 4 times the register
3770          * value.
3771          */
3772         max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
3773         if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
3774                 int curr = regulator_get_current_limit(mmc->supply.vmmc);
3775                 if (curr > 0) {
3776
3777                         /* convert to SDHCI_MAX_CURRENT format */
3778                         curr = curr/1000;  /* convert to mA */
3779                         curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
3780
3781                         curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
3782                         max_current_caps =
3783                                 (curr << SDHCI_MAX_CURRENT_330_SHIFT) |
3784                                 (curr << SDHCI_MAX_CURRENT_300_SHIFT) |
3785                                 (curr << SDHCI_MAX_CURRENT_180_SHIFT);
3786                 }
3787         }
3788
3789         if (host->caps & SDHCI_CAN_VDD_330) {
3790                 ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
3791
3792                 mmc->max_current_330 = ((max_current_caps &
3793                                    SDHCI_MAX_CURRENT_330_MASK) >>
3794                                    SDHCI_MAX_CURRENT_330_SHIFT) *
3795                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3796         }
3797         if (host->caps & SDHCI_CAN_VDD_300) {
3798                 ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
3799
3800                 mmc->max_current_300 = ((max_current_caps &
3801                                    SDHCI_MAX_CURRENT_300_MASK) >>
3802                                    SDHCI_MAX_CURRENT_300_SHIFT) *
3803                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3804         }
3805         if (host->caps & SDHCI_CAN_VDD_180) {
3806                 ocr_avail |= MMC_VDD_165_195;
3807
3808                 mmc->max_current_180 = ((max_current_caps &
3809                                    SDHCI_MAX_CURRENT_180_MASK) >>
3810                                    SDHCI_MAX_CURRENT_180_SHIFT) *
3811                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3812         }
3813
3814         /* If OCR set by host, use it instead. */
3815         if (host->ocr_mask)
3816                 ocr_avail = host->ocr_mask;
3817
3818         /* If OCR set by external regulators, give it highest prio. */
3819         if (mmc->ocr_avail)
3820                 ocr_avail = mmc->ocr_avail;
3821
3822         mmc->ocr_avail = ocr_avail;
3823         mmc->ocr_avail_sdio = ocr_avail;
3824         if (host->ocr_avail_sdio)
3825                 mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
3826         mmc->ocr_avail_sd = ocr_avail;
3827         if (host->ocr_avail_sd)
3828                 mmc->ocr_avail_sd &= host->ocr_avail_sd;
3829         else /* normal SD controllers don't support 1.8V */
3830                 mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
3831         mmc->ocr_avail_mmc = ocr_avail;
3832         if (host->ocr_avail_mmc)
3833                 mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
3834
3835         if (mmc->ocr_avail == 0) {
3836                 pr_err("%s: Hardware doesn't report any support voltages.\n",
3837                        mmc_hostname(mmc));
3838                 ret = -ENODEV;
3839                 goto unreg;
3840         }
3841
3842         if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
3843                           MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
3844                           MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
3845             (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
3846                 host->flags |= SDHCI_SIGNALING_180;
3847
3848         if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
3849                 host->flags |= SDHCI_SIGNALING_120;
3850
3851         spin_lock_init(&host->lock);
3852
3853         /*
3854          * Maximum number of sectors in one transfer. Limited by SDMA boundary
3855          * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
3856          * is less anyway.
3857          */
3858         mmc->max_req_size = 524288;
3859
3860         /*
3861          * Maximum number of segments. Depends on if the hardware
3862          * can do scatter/gather or not.
3863          */
3864         if (host->flags & SDHCI_USE_ADMA) {
3865                 mmc->max_segs = SDHCI_MAX_SEGS;
3866         } else if (host->flags & SDHCI_USE_SDMA) {
3867                 mmc->max_segs = 1;
3868                 if (swiotlb_max_segment()) {
3869                         unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
3870                                                 IO_TLB_SEGSIZE;
3871                         mmc->max_req_size = min(mmc->max_req_size,
3872                                                 max_req_size);
3873                 }
3874         } else { /* PIO */
3875                 mmc->max_segs = SDHCI_MAX_SEGS;
3876         }
3877
3878         /*
3879          * Maximum segment size. Could be one segment with the maximum number
3880          * of bytes. When doing hardware scatter/gather, each entry cannot
3881          * be larger than 64 KiB though.
3882          */
3883         if (host->flags & SDHCI_USE_ADMA) {
3884                 if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
3885                         mmc->max_seg_size = 65535;
3886                 else
3887                         mmc->max_seg_size = 65536;
3888         } else {
3889                 mmc->max_seg_size = mmc->max_req_size;
3890         }
3891
3892         /*
3893          * Maximum block size. This varies from controller to controller and
3894          * is specified in the capabilities register.
3895          */
3896         if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
3897                 mmc->max_blk_size = 2;
3898         } else {
3899                 mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
3900                                 SDHCI_MAX_BLOCK_SHIFT;
3901                 if (mmc->max_blk_size >= 3) {
3902                         pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
3903                                 mmc_hostname(mmc));
3904                         mmc->max_blk_size = 0;
3905                 }
3906         }
3907
3908         mmc->max_blk_size = 512 << mmc->max_blk_size;
3909
3910         /*
3911          * Maximum block count.
3912          */
3913         mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
3914
3915         if (mmc->max_segs == 1) {
3916                 /* This may alter mmc->*_blk_* parameters */
3917                 ret = sdhci_allocate_bounce_buffer(host);
3918                 if (ret)
3919                         return ret;
3920         }
3921
3922         return 0;
3923
3924 unreg:
3925         if (!IS_ERR(mmc->supply.vqmmc))
3926                 regulator_disable(mmc->supply.vqmmc);
3927 undma:
3928         if (host->align_buffer)
3929                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3930                                   host->adma_table_sz, host->align_buffer,
3931                                   host->align_addr);
3932         host->adma_table = NULL;
3933         host->align_buffer = NULL;
3934
3935         return ret;
3936 }
3937 EXPORT_SYMBOL_GPL(sdhci_setup_host);
3938
3939 void sdhci_cleanup_host(struct sdhci_host *host)
3940 {
3941         struct mmc_host *mmc = host->mmc;
3942
3943         if (!IS_ERR(mmc->supply.vqmmc))
3944                 regulator_disable(mmc->supply.vqmmc);
3945
3946         if (host->align_buffer)
3947                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3948                                   host->adma_table_sz, host->align_buffer,
3949                                   host->align_addr);
3950         host->adma_table = NULL;
3951         host->align_buffer = NULL;
3952 }
3953 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
3954
3955 int __sdhci_add_host(struct sdhci_host *host)
3956 {
3957         struct mmc_host *mmc = host->mmc;
3958         int ret;
3959
3960         /*
3961          * Init tasklets.
3962          */
3963         tasklet_init(&host->finish_tasklet,
3964                 sdhci_tasklet_finish, (unsigned long)host);
3965
3966         setup_timer(&host->timer, sdhci_timeout_timer, (unsigned long)host);
3967         setup_timer(&host->data_timer, sdhci_timeout_data_timer,
3968                     (unsigned long)host);
3969
3970         init_waitqueue_head(&host->buf_ready_int);
3971
3972         sdhci_init(host, 0);
3973
3974         ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
3975                                    IRQF_SHARED, mmc_hostname(mmc), host);
3976         if (ret) {
3977                 pr_err("%s: Failed to request IRQ %d: %d\n",
3978                        mmc_hostname(mmc), host->irq, ret);
3979                 goto untasklet;
3980         }
3981
3982         ret = sdhci_led_register(host);
3983         if (ret) {
3984                 pr_err("%s: Failed to register LED device: %d\n",
3985                        mmc_hostname(mmc), ret);
3986                 goto unirq;
3987         }
3988
3989         mmiowb();
3990
3991         ret = mmc_add_host(mmc);
3992         if (ret)
3993                 goto unled;
3994
3995         pr_info("%s: SDHCI controller on %s [%s] using %s\n",
3996                 mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
3997                 (host->flags & SDHCI_USE_ADMA) ?
3998                 (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
3999                 (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4000
4001         sdhci_enable_card_detection(host);
4002
4003         return 0;
4004
4005 unled:
4006         sdhci_led_unregister(host);
4007 unirq:
4008         sdhci_do_reset(host, SDHCI_RESET_ALL);
4009         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4010         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4011         free_irq(host->irq, host);
4012 untasklet:
4013         tasklet_kill(&host->finish_tasklet);
4014
4015         return ret;
4016 }
4017 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4018
4019 int sdhci_add_host(struct sdhci_host *host)
4020 {
4021         int ret;
4022
4023         ret = sdhci_setup_host(host);
4024         if (ret)
4025                 return ret;
4026
4027         ret = __sdhci_add_host(host);
4028         if (ret)
4029                 goto cleanup;
4030
4031         return 0;
4032
4033 cleanup:
4034         sdhci_cleanup_host(host);
4035
4036         return ret;
4037 }
4038 EXPORT_SYMBOL_GPL(sdhci_add_host);
4039
4040 void sdhci_remove_host(struct sdhci_host *host, int dead)
4041 {
4042         struct mmc_host *mmc = host->mmc;
4043         unsigned long flags;
4044
4045         if (dead) {
4046                 spin_lock_irqsave(&host->lock, flags);
4047
4048                 host->flags |= SDHCI_DEVICE_DEAD;
4049
4050                 if (sdhci_has_requests(host)) {
4051                         pr_err("%s: Controller removed during "
4052                                 " transfer!\n", mmc_hostname(mmc));
4053                         sdhci_error_out_mrqs(host, -ENOMEDIUM);
4054                 }
4055
4056                 spin_unlock_irqrestore(&host->lock, flags);
4057         }
4058
4059         sdhci_disable_card_detection(host);
4060
4061         mmc_remove_host(mmc);
4062
4063         sdhci_led_unregister(host);
4064
4065         if (!dead)
4066                 sdhci_do_reset(host, SDHCI_RESET_ALL);
4067
4068         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4069         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4070         free_irq(host->irq, host);
4071
4072         del_timer_sync(&host->timer);
4073         del_timer_sync(&host->data_timer);
4074
4075         tasklet_kill(&host->finish_tasklet);
4076
4077         if (!IS_ERR(mmc->supply.vqmmc))
4078                 regulator_disable(mmc->supply.vqmmc);
4079
4080         if (host->align_buffer)
4081                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4082                                   host->adma_table_sz, host->align_buffer,
4083                                   host->align_addr);
4084
4085         host->adma_table = NULL;
4086         host->align_buffer = NULL;
4087 }
4088
4089 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4090
4091 void sdhci_free_host(struct sdhci_host *host)
4092 {
4093         mmc_free_host(host->mmc);
4094 }
4095
4096 EXPORT_SYMBOL_GPL(sdhci_free_host);
4097
4098 /*****************************************************************************\
4099  *                                                                           *
4100  * Driver init/exit                                                          *
4101  *                                                                           *
4102 \*****************************************************************************/
4103
4104 static int __init sdhci_drv_init(void)
4105 {
4106         pr_info(DRIVER_NAME
4107                 ": Secure Digital Host Controller Interface driver\n");
4108         pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4109
4110         return 0;
4111 }
4112
4113 static void __exit sdhci_drv_exit(void)
4114 {
4115 }
4116
4117 module_init(sdhci_drv_init);
4118 module_exit(sdhci_drv_exit);
4119
4120 module_param(debug_quirks, uint, 0444);
4121 module_param(debug_quirks2, uint, 0444);
4122
4123 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4124 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4125 MODULE_LICENSE("GPL");
4126
4127 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4128 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");