drivers/mmc/host/meson-mx-sdhc-mmc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Amlogic Meson6/Meson8/Meson8b/Meson8m2 SDHC MMC host controller driver.
   4  *
   5  * Copyright (C) 2020 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
   6  */
   7
   8 #include <linux/clk.h>
   9 #include <linux/device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/iopoll.h>
  13 #include <linux/module.h>
  14 #include <linux/of.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/property.h>
  17 #include <linux/regmap.h>
  18 #include <linux/regulator/consumer.h>
  19 #include <linux/types.h>
  20
  21 #include <linux/mmc/host.h>
  22 #include <linux/mmc/mmc.h>
  23 #include <linux/mmc/sdio.h>
  24 #include <linux/mmc/slot-gpio.h>
  25
  26 #include "meson-mx-sdhc.h"
  27
  28 #define MESON_SDHC_NUM_BULK_CLKS                                4
  29 #define MESON_SDHC_MAX_BLK_SIZE                                 512
  30 #define MESON_SDHC_NUM_TUNING_TRIES                             10
  31
  32 #define MESON_SDHC_WAIT_CMD_READY_SLEEP_US                      1
  33 #define MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US                    100000
  34 #define MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US                    1
  35 #define MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US                  200
  36
  37 struct meson_mx_sdhc_data {
  38         void            (*init_hw)(struct mmc_host *mmc);
  39         void            (*set_pdma)(struct mmc_host *mmc);
  40         void            (*wait_before_send)(struct mmc_host *mmc);
  41         bool            hardware_flush_all_cmds;
  42 };
  43
  44 struct meson_mx_sdhc_host {
  45         struct mmc_host                 *mmc;
  46
  47         struct mmc_request              *mrq;
  48         struct mmc_command              *cmd;
  49         int                             error;
  50
  51         struct regmap                   *regmap;
  52
  53         struct clk                      *pclk;
  54         struct clk                      *sd_clk;
  55         struct clk_bulk_data            bulk_clks[MESON_SDHC_NUM_BULK_CLKS];
  56         bool                            bulk_clks_enabled;
  57
  58         const struct meson_mx_sdhc_data *platform;
  59 };
  60
  61 static const struct regmap_config meson_mx_sdhc_regmap_config = {
  62         .reg_bits = 8,
  63         .val_bits = 32,
  64         .reg_stride = 4,
  65         .max_register = MESON_SDHC_CLK2,
  66 };
  67
  68 static void meson_mx_sdhc_hw_reset(struct mmc_host *mmc)
  69 {
  70         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
  71
  72         regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_MAIN_CTRL |
  73                      MESON_SDHC_SRST_RXFIFO | MESON_SDHC_SRST_TXFIFO |
  74                      MESON_SDHC_SRST_DPHY_RX | MESON_SDHC_SRST_DPHY_TX |
  75                      MESON_SDHC_SRST_DMA_IF);
  76         usleep_range(10, 100);
  77
  78         regmap_write(host->regmap, MESON_SDHC_SRST, 0);
  79         usleep_range(10, 100);
  80 }
  81
  82 static void meson_mx_sdhc_clear_fifo(struct mmc_host *mmc)
  83 {
  84         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
  85         u32 stat;
  86
  87         regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
  88         if (!FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat) &&
  89             !FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat))
  90                 return;
  91
  92         regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_RXFIFO |
  93                      MESON_SDHC_SRST_TXFIFO | MESON_SDHC_SRST_MAIN_CTRL);
  94         udelay(5);
  95
  96         regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
  97         if (FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat) ||
  98             FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat))
  99                 dev_warn(mmc_dev(host->mmc),
 100                          "Failed to clear FIFOs, RX: %lu, TX: %lu\n",
 101                          FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat),
 102                          FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat));
 103 }
 104
 105 static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc)
 106 {
 107         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 108         u32 stat, esta;
 109         int ret;
 110
 111         ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_STAT, stat,
 112                                        !(stat & MESON_SDHC_STAT_CMD_BUSY),
 113                                        MESON_SDHC_WAIT_CMD_READY_SLEEP_US,
 114                                        MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US);
 115         if (ret) {
 116                 dev_warn(mmc_dev(mmc),
 117                          "Failed to poll for CMD_BUSY while processing CMD%d\n",
 118                          host->cmd->opcode);
 119                 meson_mx_sdhc_hw_reset(mmc);
 120         }
 121
 122         ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, esta,
 123                                        !(esta & MESON_SDHC_ESTA_11_13),
 124                                        MESON_SDHC_WAIT_CMD_READY_SLEEP_US,
 125                                        MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US);
 126         if (ret) {
 127                 dev_warn(mmc_dev(mmc),
 128                          "Failed to poll for ESTA[13:11] while processing CMD%d\n",
 129                          host->cmd->opcode);
 130                 meson_mx_sdhc_hw_reset(mmc);
 131         }
 132 }
 133
 134 static void meson_mx_sdhc_start_cmd(struct mmc_host *mmc,
 135                                     struct mmc_command *cmd)
 136 {
 137         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 138         bool manual_stop = false;
 139         u32 ictl, send;
 140         int pack_len;
 141
 142         host->cmd = cmd;
 143
 144         ictl = MESON_SDHC_ICTL_DATA_TIMEOUT | MESON_SDHC_ICTL_DATA_ERR_CRC |
 145                MESON_SDHC_ICTL_RXFIFO_FULL | MESON_SDHC_ICTL_TXFIFO_EMPTY |
 146                MESON_SDHC_ICTL_RESP_TIMEOUT | MESON_SDHC_ICTL_RESP_ERR_CRC;
 147
 148         send = FIELD_PREP(MESON_SDHC_SEND_CMD_INDEX, cmd->opcode);
 149
 150         if (cmd->data) {
 151                 send |= MESON_SDHC_SEND_CMD_HAS_DATA;
 152                 send |= FIELD_PREP(MESON_SDHC_SEND_TOTAL_PACK,
 153                                    cmd->data->blocks - 1);
 154
 155                 if (cmd->data->blksz < MESON_SDHC_MAX_BLK_SIZE)
 156                         pack_len = cmd->data->blksz;
 157                 else
 158                         pack_len = 0;
 159
 160                 if (cmd->data->flags & MMC_DATA_WRITE)
 161                         send |= MESON_SDHC_SEND_DATA_DIR;
 162
 163                 /*
 164                  * If command with no data, just wait response done
 165                  * interrupt(int[0]), and if command with data transfer, just
 166                  * wait dma done interrupt(int[11]), don't need care about
 167                  * dat0 busy or not.
 168                  */
 169                 if (host->platform->hardware_flush_all_cmds ||
 170                     cmd->data->flags & MMC_DATA_WRITE)
 171                         /* hardware flush: */
 172                         ictl |= MESON_SDHC_ICTL_DMA_DONE;
 173                 else
 174                         /* software flush: */
 175                         ictl |= MESON_SDHC_ICTL_DATA_XFER_OK;
 176
 177                 /*
 178                  * Mimic the logic from the vendor driver where (only)
 179                  * SD_IO_RW_EXTENDED commands with more than one block set the
 180                  * MESON_SDHC_MISC_MANUAL_STOP bit. This fixes the firmware
 181                  * download in the brcmfmac driver for a BCM43362/1 card.
 182                  * Without this sdio_memcpy_toio() (with a size of 219557
 183                  * bytes) times out if MESON_SDHC_MISC_MANUAL_STOP is not set.
 184                  */
 185                 manual_stop = cmd->data->blocks > 1 &&
 186                               cmd->opcode == SD_IO_RW_EXTENDED;
 187         } else {
 188                 pack_len = 0;
 189
 190                 ictl |= MESON_SDHC_ICTL_RESP_OK;
 191         }
 192
 193         regmap_update_bits(host->regmap, MESON_SDHC_MISC,
 194                            MESON_SDHC_MISC_MANUAL_STOP,
 195                            manual_stop ? MESON_SDHC_MISC_MANUAL_STOP : 0);
 196
 197         if (cmd->opcode == MMC_STOP_TRANSMISSION)
 198                 send |= MESON_SDHC_SEND_DATA_STOP;
 199
 200         if (cmd->flags & MMC_RSP_PRESENT)
 201                 send |= MESON_SDHC_SEND_CMD_HAS_RESP;
 202
 203         if (cmd->flags & MMC_RSP_136) {
 204                 send |= MESON_SDHC_SEND_RESP_LEN;
 205                 send |= MESON_SDHC_SEND_RESP_NO_CRC;
 206         }
 207
 208         if (!(cmd->flags & MMC_RSP_CRC))
 209                 send |= MESON_SDHC_SEND_RESP_NO_CRC;
 210
 211         if (cmd->flags & MMC_RSP_BUSY)
 212                 send |= MESON_SDHC_SEND_R1B;
 213
 214         /* enable the new IRQs and mask all pending ones */
 215         regmap_write(host->regmap, MESON_SDHC_ICTL, ictl);
 216         regmap_write(host->regmap, MESON_SDHC_ISTA, MESON_SDHC_ISTA_ALL_IRQS);
 217
 218         regmap_write(host->regmap, MESON_SDHC_ARGU, cmd->arg);
 219
 220         regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
 221                            MESON_SDHC_CTRL_PACK_LEN,
 222                            FIELD_PREP(MESON_SDHC_CTRL_PACK_LEN, pack_len));
 223
 224         if (cmd->data)
 225                 regmap_write(host->regmap, MESON_SDHC_ADDR,
 226                              sg_dma_address(cmd->data->sg));
 227
 228         meson_mx_sdhc_wait_cmd_ready(mmc);
 229
 230         if (cmd->data)
 231                 host->platform->set_pdma(mmc);
 232
 233         if (host->platform->wait_before_send)
 234                 host->platform->wait_before_send(mmc);
 235
 236         regmap_write(host->regmap, MESON_SDHC_SEND, send);
 237 }
 238
 239 static void meson_mx_sdhc_disable_clks(struct mmc_host *mmc)
 240 {
 241         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 242
 243         if (!host->bulk_clks_enabled)
 244                 return;
 245
 246         clk_bulk_disable_unprepare(MESON_SDHC_NUM_BULK_CLKS, host->bulk_clks);
 247
 248         host->bulk_clks_enabled = false;
 249 }
 250
 251 static int meson_mx_sdhc_enable_clks(struct mmc_host *mmc)
 252 {
 253         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 254         int ret;
 255
 256         if (host->bulk_clks_enabled)
 257                 return 0;
 258
 259         ret = clk_bulk_prepare_enable(MESON_SDHC_NUM_BULK_CLKS,
 260                                       host->bulk_clks);
 261         if (ret)
 262                 return ret;
 263
 264         host->bulk_clks_enabled = true;
 265
 266         return 0;
 267 }
 268
 269 static int meson_mx_sdhc_set_clk(struct mmc_host *mmc, struct mmc_ios *ios)
 270 {
 271         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 272         u32 rx_clk_phase;
 273         int ret;
 274
 275         meson_mx_sdhc_disable_clks(mmc);
 276
 277         if (ios->clock) {
 278                 ret = clk_set_rate(host->sd_clk, ios->clock);
 279                 if (ret) {
 280                         dev_warn(mmc_dev(mmc),
 281                                  "Failed to set MMC clock to %uHz: %d\n",
 282                                  ios->clock, host->error);
 283                         return ret;
 284                 }
 285
 286                 ret = meson_mx_sdhc_enable_clks(mmc);
 287                 if (ret)
 288                         return ret;
 289
 290                 mmc->actual_clock = clk_get_rate(host->sd_clk);
 291
 292                 /*
 293                  * according to Amlogic the following latching points are
 294                  * selected with empirical values, there is no (known) formula
 295                  * to calculate these.
 296                  */
 297                 if (mmc->actual_clock > 100000000) {
 298                         rx_clk_phase = 1;
 299                 } else if (mmc->actual_clock > 45000000) {
 300                         if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
 301                                 rx_clk_phase = 15;
 302                         else
 303                                 rx_clk_phase = 11;
 304                 } else if (mmc->actual_clock >= 25000000) {
 305                         rx_clk_phase = 15;
 306                 } else if (mmc->actual_clock > 5000000) {
 307                         rx_clk_phase = 23;
 308                 } else if (mmc->actual_clock > 1000000) {
 309                         rx_clk_phase = 55;
 310                 } else {
 311                         rx_clk_phase = 1061;
 312                 }
 313
 314                 regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
 315                                    MESON_SDHC_CLK2_RX_CLK_PHASE,
 316                                    FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
 317                                               rx_clk_phase));
 318         } else {
 319                 mmc->actual_clock = 0;
 320         }
 321
 322         return 0;
 323 }
 324
 325 static void meson_mx_sdhc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 326 {
 327         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 328         unsigned short vdd = ios->vdd;
 329
 330         switch (ios->power_mode) {
 331         case MMC_POWER_OFF:
 332                 vdd = 0;
 333                 fallthrough;
 334
 335         case MMC_POWER_UP:
 336                 if (!IS_ERR(mmc->supply.vmmc)) {
 337                         host->error = mmc_regulator_set_ocr(mmc,
 338                                                             mmc->supply.vmmc,
 339                                                             vdd);
 340                         if (host->error)
 341                                 return;
 342                 }
 343
 344                 break;
 345
 346         case MMC_POWER_ON:
 347                 break;
 348         }
 349
 350         host->error = meson_mx_sdhc_set_clk(mmc, ios);
 351         if (host->error)
 352                 return;
 353
 354         switch (ios->bus_width) {
 355         case MMC_BUS_WIDTH_1:
 356                 regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
 357                                    MESON_SDHC_CTRL_DAT_TYPE,
 358                                    FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 0));
 359                 break;
 360
 361         case MMC_BUS_WIDTH_4:
 362                 regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
 363                                    MESON_SDHC_CTRL_DAT_TYPE,
 364                                    FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 1));
 365                 break;
 366
 367         case MMC_BUS_WIDTH_8:
 368                 regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
 369                                    MESON_SDHC_CTRL_DAT_TYPE,
 370                                    FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 2));
 371                 break;
 372
 373         default:
 374                 dev_err(mmc_dev(mmc), "unsupported bus width: %d\n",
 375                         ios->bus_width);
 376                 host->error = -EINVAL;
 377                 return;
 378         }
 379 }
 380
 381 static int meson_mx_sdhc_map_dma(struct mmc_host *mmc, struct mmc_request *mrq)
 382 {
 383         struct mmc_data *data = mrq->data;
 384         unsigned int dma_len;
 385
 386         if (!data)
 387                 return 0;
 388
 389         dma_len = dma_map_sg(mmc_dev(mmc), data->sg, data->sg_len,
 390                              mmc_get_dma_dir(data));
 391         if (!dma_len) {
 392                 dev_err(mmc_dev(mmc), "dma_map_sg failed\n");
 393                 return -ENOMEM;
 394         }
 395
 396         return 0;
 397 }
 398
 399 static void meson_mx_sdhc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 400 {
 401         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 402         struct mmc_command *cmd = mrq->cmd;
 403
 404         if (!host->error)
 405                 host->error = meson_mx_sdhc_map_dma(mmc, mrq);
 406
 407         if (host->error) {
 408                 cmd->error = host->error;
 409                 mmc_request_done(mmc, mrq);
 410                 return;
 411         }
 412
 413         host->mrq = mrq;
 414
 415         meson_mx_sdhc_start_cmd(mmc, mrq->cmd);
 416 }
 417
 418 static int meson_mx_sdhc_card_busy(struct mmc_host *mmc)
 419 {
 420         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 421         u32 stat;
 422
 423         regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
 424         return FIELD_GET(MESON_SDHC_STAT_DAT3_0, stat) == 0;
 425 }
 426
 427 static bool meson_mx_sdhc_tuning_point_matches(struct mmc_host *mmc,
 428                                                u32 opcode)
 429 {
 430         unsigned int i, num_matches = 0;
 431         int ret;
 432
 433         for (i = 0; i < MESON_SDHC_NUM_TUNING_TRIES; i++) {
 434                 ret = mmc_send_tuning(mmc, opcode, NULL);
 435                 if (!ret)
 436                         num_matches++;
 437         }
 438
 439         return num_matches == MESON_SDHC_NUM_TUNING_TRIES;
 440 }
 441
 442 static int meson_mx_sdhc_execute_tuning(struct mmc_host *mmc, u32 opcode)
 443 {
 444         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 445         int div, start, len, best_start, best_len;
 446         int curr_phase, old_phase, new_phase;
 447         u32 val;
 448
 449         len = 0;
 450         start = 0;
 451         best_len = 0;
 452
 453         regmap_read(host->regmap, MESON_SDHC_CLK2, &val);
 454         old_phase = FIELD_GET(MESON_SDHC_CLK2_RX_CLK_PHASE, val);
 455
 456         regmap_read(host->regmap, MESON_SDHC_CLKC, &val);
 457         div = FIELD_GET(MESON_SDHC_CLKC_CLK_DIV, val);
 458
 459         for (curr_phase = 0; curr_phase <= div; curr_phase++) {
 460                 regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
 461                                    MESON_SDHC_CLK2_RX_CLK_PHASE,
 462                                    FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
 463                                               curr_phase));
 464
 465                 if (meson_mx_sdhc_tuning_point_matches(mmc, opcode)) {
 466                         if (!len) {
 467                                 start = curr_phase;
 468
 469                                 dev_dbg(mmc_dev(mmc),
 470                                         "New RX phase window starts at %u\n",
 471                                         start);
 472                         }
 473
 474                         len++;
 475                 } else {
 476                         if (len > best_len) {
 477                                 best_start = start;
 478                                 best_len = len;
 479
 480                                 dev_dbg(mmc_dev(mmc),
 481                                         "New best RX phase window: %u - %u\n",
 482                                         best_start, best_start + best_len);
 483                         }
 484
 485                         /* reset the current window */
 486                         len = 0;
 487                 }
 488         }
 489
 490         if (len > best_len)
 491                 /* the last window is the best (or possibly only) window */
 492                 new_phase = start + (len / 2);
 493         else if (best_len)
 494                 /* there was a better window than the last */
 495                 new_phase = best_start + (best_len / 2);
 496         else
 497                 /* no window was found at all, reset to the original phase */
 498                 new_phase = old_phase;
 499
 500         regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
 501                            MESON_SDHC_CLK2_RX_CLK_PHASE,
 502                            FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
 503                                       new_phase));
 504
 505         if (!len && !best_len)
 506                 return -EIO;
 507
 508         dev_dbg(mmc_dev(mmc), "Tuned RX clock phase to %u\n", new_phase);
 509
 510         return 0;
 511 }
 512
 513 static const struct mmc_host_ops meson_mx_sdhc_ops = {
 514         .card_hw_reset                  = meson_mx_sdhc_hw_reset,
 515         .request                        = meson_mx_sdhc_request,
 516         .set_ios                        = meson_mx_sdhc_set_ios,
 517         .card_busy                      = meson_mx_sdhc_card_busy,
 518         .execute_tuning                 = meson_mx_sdhc_execute_tuning,
 519         .get_cd                         = mmc_gpio_get_cd,
 520         .get_ro                         = mmc_gpio_get_ro,
 521 };
 522
 523 static void meson_mx_sdhc_request_done(struct meson_mx_sdhc_host *host)
 524 {
 525         struct mmc_request *mrq = host->mrq;
 526         struct mmc_host *mmc = host->mmc;
 527
 528         /* disable interrupts and mask all pending ones */
 529         regmap_update_bits(host->regmap, MESON_SDHC_ICTL,
 530                            MESON_SDHC_ICTL_ALL_IRQS, 0);
 531         regmap_update_bits(host->regmap, MESON_SDHC_ISTA,
 532                            MESON_SDHC_ISTA_ALL_IRQS, MESON_SDHC_ISTA_ALL_IRQS);
 533
 534         host->mrq = NULL;
 535         host->cmd = NULL;
 536
 537         mmc_request_done(mmc, mrq);
 538 }
 539
 540 static u32 meson_mx_sdhc_read_response(struct meson_mx_sdhc_host *host, u8 idx)
 541 {
 542         u32 val;
 543
 544         regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 545                            MESON_SDHC_PDMA_DMA_MODE, 0);
 546
 547         regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 548                            MESON_SDHC_PDMA_PIO_RDRESP,
 549                            FIELD_PREP(MESON_SDHC_PDMA_PIO_RDRESP, idx));
 550
 551         regmap_read(host->regmap, MESON_SDHC_ARGU, &val);
 552
 553         return val;
 554 }
 555
 556 static irqreturn_t meson_mx_sdhc_irq(int irq, void *data)
 557 {
 558         struct meson_mx_sdhc_host *host = data;
 559         struct mmc_command *cmd = host->cmd;
 560         u32 ictl, ista;
 561
 562         regmap_read(host->regmap, MESON_SDHC_ICTL, &ictl);
 563         regmap_read(host->regmap, MESON_SDHC_ISTA, &ista);
 564
 565         if (!(ictl & ista))
 566                 return IRQ_NONE;
 567
 568         if (ista & MESON_SDHC_ISTA_RXFIFO_FULL ||
 569             ista & MESON_SDHC_ISTA_TXFIFO_EMPTY)
 570                 cmd->error = -EIO;
 571         else if (ista & MESON_SDHC_ISTA_RESP_ERR_CRC)
 572                 cmd->error = -EILSEQ;
 573         else if (ista & MESON_SDHC_ISTA_RESP_TIMEOUT)
 574                 cmd->error = -ETIMEDOUT;
 575
 576         if (cmd->data) {
 577                 if (ista & MESON_SDHC_ISTA_DATA_ERR_CRC)
 578                         cmd->data->error = -EILSEQ;
 579                 else if (ista & MESON_SDHC_ISTA_DATA_TIMEOUT)
 580                         cmd->data->error = -ETIMEDOUT;
 581         }
 582
 583         if (cmd->error || (cmd->data && cmd->data->error))
 584                 dev_dbg(mmc_dev(host->mmc), "CMD%d error, ISTA: 0x%08x\n",
 585                         cmd->opcode, ista);
 586
 587         return IRQ_WAKE_THREAD;
 588 }
 589
 590 static irqreturn_t meson_mx_sdhc_irq_thread(int irq, void *irq_data)
 591 {
 592         struct meson_mx_sdhc_host *host = irq_data;
 593         struct mmc_command *cmd;
 594         u32 val;
 595
 596         cmd = host->cmd;
 597         if (WARN_ON(!cmd))
 598                 return IRQ_HANDLED;
 599
 600         if (cmd->data && !cmd->data->error) {
 601                 if (!host->platform->hardware_flush_all_cmds &&
 602                     cmd->data->flags & MMC_DATA_READ) {
 603                         meson_mx_sdhc_wait_cmd_ready(host->mmc);
 604
 605                         /*
 606                          * If MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH was
 607                          * previously 0x1 then it has to be set to 0x3. If it
 608                          * was 0x0 before then it has to be set to 0x2. Without
 609                          * this reading SD cards sometimes transfers garbage,
 610                          * which results in cards not being detected due to:
 611                          *   unrecognised SCR structure version <random number>
 612                          */
 613                         val = FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
 614                                          2);
 615                         regmap_update_bits(host->regmap, MESON_SDHC_PDMA, val,
 616                                            val);
 617                 }
 618
 619                 dma_unmap_sg(mmc_dev(host->mmc), cmd->data->sg,
 620                              cmd->data->sg_len, mmc_get_dma_dir(cmd->data));
 621
 622                 cmd->data->bytes_xfered = cmd->data->blksz * cmd->data->blocks;
 623         }
 624
 625         meson_mx_sdhc_wait_cmd_ready(host->mmc);
 626
 627         if (cmd->flags & MMC_RSP_136) {
 628                 cmd->resp[0] = meson_mx_sdhc_read_response(host, 4);
 629                 cmd->resp[1] = meson_mx_sdhc_read_response(host, 3);
 630                 cmd->resp[2] = meson_mx_sdhc_read_response(host, 2);
 631                 cmd->resp[3] = meson_mx_sdhc_read_response(host, 1);
 632         } else {
 633                 cmd->resp[0] = meson_mx_sdhc_read_response(host, 0);
 634         }
 635
 636         if (cmd->error == -EIO || cmd->error == -ETIMEDOUT)
 637                 meson_mx_sdhc_hw_reset(host->mmc);
 638         else if (cmd->data)
 639                 /*
 640                  * Clear the FIFOs after completing data transfers to prevent
 641                  * corrupting data on write access. It's not clear why this is
 642                  * needed (for reads and writes), but it mimics what the BSP
 643                  * kernel did.
 644                  */
 645                 meson_mx_sdhc_clear_fifo(host->mmc);
 646
 647         meson_mx_sdhc_request_done(host);
 648
 649         return IRQ_HANDLED;
 650 }
 651
 652 static void meson_mx_sdhc_init_hw_meson8(struct mmc_host *mmc)
 653 {
 654         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 655
 656         regmap_write(host->regmap, MESON_SDHC_MISC,
 657                      FIELD_PREP(MESON_SDHC_MISC_TXSTART_THRES, 7) |
 658                      FIELD_PREP(MESON_SDHC_MISC_WCRC_ERR_PATT, 5) |
 659                      FIELD_PREP(MESON_SDHC_MISC_WCRC_OK_PATT, 2));
 660
 661         regmap_write(host->regmap, MESON_SDHC_ENHC,
 662                      FIELD_PREP(MESON_SDHC_ENHC_RXFIFO_TH, 63) |
 663                      MESON_SDHC_ENHC_MESON6_DMA_WR_RESP |
 664                      FIELD_PREP(MESON_SDHC_ENHC_MESON6_RX_TIMEOUT, 255) |
 665                      FIELD_PREP(MESON_SDHC_ENHC_SDIO_IRQ_PERIOD, 12));
 666 };
 667
 668 static void meson_mx_sdhc_set_pdma_meson8(struct mmc_host *mmc)
 669 {
 670         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 671
 672         if (host->cmd->data->flags & MMC_DATA_WRITE)
 673                 regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 674                                    MESON_SDHC_PDMA_DMA_MODE |
 675                                    MESON_SDHC_PDMA_RD_BURST |
 676                                    MESON_SDHC_PDMA_TXFIFO_FILL,
 677                                    MESON_SDHC_PDMA_DMA_MODE |
 678                                    FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 31) |
 679                                    MESON_SDHC_PDMA_TXFIFO_FILL);
 680         else
 681                 regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 682                                    MESON_SDHC_PDMA_DMA_MODE |
 683                                    MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
 684                                    MESON_SDHC_PDMA_DMA_MODE |
 685                                    FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
 686                                               1));
 687
 688         if (host->cmd->data->flags & MMC_DATA_WRITE)
 689                 regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 690                                    MESON_SDHC_PDMA_RD_BURST,
 691                                    FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 15));
 692 }
 693
 694 static void meson_mx_sdhc_wait_before_send_meson8(struct mmc_host *mmc)
 695 {
 696         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 697         u32 val;
 698         int ret;
 699
 700         ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, val,
 701                                        val == 0,
 702                                        MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US,
 703                                        MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US);
 704         if (ret)
 705                 dev_warn(mmc_dev(mmc),
 706                          "Failed to wait for ESTA to clear: 0x%08x\n", val);
 707
 708         if (host->cmd->data && host->cmd->data->flags & MMC_DATA_WRITE) {
 709                 ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_STAT,
 710                                         val, val & MESON_SDHC_STAT_TXFIFO_CNT,
 711                                         MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US,
 712                                         MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US);
 713                 if (ret)
 714                         dev_warn(mmc_dev(mmc),
 715                                  "Failed to wait for TX FIFO to fill\n");
 716         }
 717 }
 718
 719 static void meson_mx_sdhc_init_hw_meson8m2(struct mmc_host *mmc)
 720 {
 721         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 722
 723         regmap_write(host->regmap, MESON_SDHC_MISC,
 724                      FIELD_PREP(MESON_SDHC_MISC_TXSTART_THRES, 6) |
 725                      FIELD_PREP(MESON_SDHC_MISC_WCRC_ERR_PATT, 5) |
 726                      FIELD_PREP(MESON_SDHC_MISC_WCRC_OK_PATT, 2));
 727
 728         regmap_write(host->regmap, MESON_SDHC_ENHC,
 729                      FIELD_PREP(MESON_SDHC_ENHC_RXFIFO_TH, 64) |
 730                      FIELD_PREP(MESON_SDHC_ENHC_MESON8M2_DEBUG, 1) |
 731                      MESON_SDHC_ENHC_MESON8M2_WRRSP_MODE |
 732                      FIELD_PREP(MESON_SDHC_ENHC_SDIO_IRQ_PERIOD, 12));
 733 }
 734
 735 static void meson_mx_sdhc_set_pdma_meson8m2(struct mmc_host *mmc)
 736 {
 737         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 738
 739         regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
 740                            MESON_SDHC_PDMA_DMA_MODE, MESON_SDHC_PDMA_DMA_MODE);
 741 }
 742
 743 static void meson_mx_sdhc_init_hw(struct mmc_host *mmc)
 744 {
 745         struct meson_mx_sdhc_host *host = mmc_priv(mmc);
 746
 747         meson_mx_sdhc_hw_reset(mmc);
 748
 749         regmap_write(host->regmap, MESON_SDHC_CTRL,
 750                      FIELD_PREP(MESON_SDHC_CTRL_RX_PERIOD, 0xf) |
 751                      FIELD_PREP(MESON_SDHC_CTRL_RX_TIMEOUT, 0x7f) |
 752                      FIELD_PREP(MESON_SDHC_CTRL_RX_ENDIAN, 0x7) |
 753                      FIELD_PREP(MESON_SDHC_CTRL_TX_ENDIAN, 0x7));
 754
 755         /*
 756          * start with a valid divider and enable the memory (un-setting
 757          * MESON_SDHC_CLKC_MEM_PWR_OFF).
 758          */
 759         regmap_write(host->regmap, MESON_SDHC_CLKC, MESON_SDHC_CLKC_CLK_DIV);
 760
 761         regmap_write(host->regmap, MESON_SDHC_CLK2,
 762                      FIELD_PREP(MESON_SDHC_CLK2_SD_CLK_PHASE, 1));
 763
 764         regmap_write(host->regmap, MESON_SDHC_PDMA,
 765                      MESON_SDHC_PDMA_DMA_URGENT |
 766                      FIELD_PREP(MESON_SDHC_PDMA_WR_BURST, 7) |
 767                      FIELD_PREP(MESON_SDHC_PDMA_TXFIFO_TH, 49) |
 768                      FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 15) |
 769                      FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_TH, 7));
 770
 771         /* some initialization bits depend on the SoC: */
 772         host->platform->init_hw(mmc);
 773
 774         /* disable and mask all interrupts: */
 775         regmap_write(host->regmap, MESON_SDHC_ICTL, 0);
 776         regmap_write(host->regmap, MESON_SDHC_ISTA, MESON_SDHC_ISTA_ALL_IRQS);
 777 }
 778
 779 static void meason_mx_mmc_free_host(void *data)
 780 {
 781        mmc_free_host(data);
 782 }
 783
 784 static int meson_mx_sdhc_probe(struct platform_device *pdev)
 785 {
 786         struct device *dev = &pdev->dev;
 787         struct meson_mx_sdhc_host *host;
 788         struct mmc_host *mmc;
 789         void __iomem *base;
 790         int ret, irq;
 791
 792         mmc = mmc_alloc_host(sizeof(*host), dev);
 793         if (!mmc)
 794                 return -ENOMEM;
 795
 796         ret = devm_add_action_or_reset(dev, meason_mx_mmc_free_host, mmc);
 797         if (ret) {
 798                 dev_err(dev, "Failed to register mmc_free_host action\n");
 799                 return ret;
 800         }
 801
 802         host = mmc_priv(mmc);
 803         host->mmc = mmc;
 804
 805         platform_set_drvdata(pdev, host);
 806
 807         host->platform = device_get_match_data(dev);
 808         if (!host->platform)
 809                 return -EINVAL;
 810
 811         base = devm_platform_ioremap_resource(pdev, 0);
 812         if (IS_ERR(base))
 813                 return PTR_ERR(base);
 814
 815         host->regmap = devm_regmap_init_mmio(dev, base,
 816                                              &meson_mx_sdhc_regmap_config);
 817         if (IS_ERR(host->regmap))
 818                 return PTR_ERR(host->regmap);
 819
 820         host->pclk = devm_clk_get(dev, "pclk");
 821         if (IS_ERR(host->pclk))
 822                 return PTR_ERR(host->pclk);
 823
 824         /* accessing any register requires the module clock to be enabled: */
 825         ret = clk_prepare_enable(host->pclk);
 826         if (ret) {
 827                 dev_err(dev, "Failed to enable 'pclk' clock\n");
 828                 return ret;
 829         }
 830
 831         meson_mx_sdhc_init_hw(mmc);
 832
 833         ret = meson_mx_sdhc_register_clkc(dev, base, host->bulk_clks);
 834         if (ret)
 835                 goto err_disable_pclk;
 836
 837         host->sd_clk = host->bulk_clks[1].clk;
 838
 839         /* Get regulators and the supported OCR mask */
 840         ret = mmc_regulator_get_supply(mmc);
 841         if (ret)
 842                 goto err_disable_pclk;
 843
 844         mmc->max_req_size = SZ_128K;
 845         mmc->max_seg_size = mmc->max_req_size;
 846         mmc->max_blk_count = FIELD_GET(MESON_SDHC_SEND_TOTAL_PACK, ~0);
 847         mmc->max_blk_size = MESON_SDHC_MAX_BLK_SIZE;
 848         mmc->max_busy_timeout = 30 * MSEC_PER_SEC;
 849         mmc->f_min = clk_round_rate(host->sd_clk, 1);
 850         mmc->f_max = clk_round_rate(host->sd_clk, ULONG_MAX);
 851         mmc->max_current_180 = 300;
 852         mmc->max_current_330 = 300;
 853         mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_HW_RESET;
 854         mmc->ops = &meson_mx_sdhc_ops;
 855
 856         ret = mmc_of_parse(mmc);
 857         if (ret)
 858                 goto err_disable_pclk;
 859
 860         irq = platform_get_irq(pdev, 0);
 861         if (irq < 0) {
 862                 ret = irq;
 863                 goto err_disable_pclk;
 864         }
 865
 866         ret = devm_request_threaded_irq(dev, irq, meson_mx_sdhc_irq,
 867                                         meson_mx_sdhc_irq_thread, IRQF_ONESHOT,
 868                                         NULL, host);
 869         if (ret)
 870                 goto err_disable_pclk;
 871
 872         ret = mmc_add_host(mmc);
 873         if (ret)
 874                 goto err_disable_pclk;
 875
 876         return 0;
 877
 878 err_disable_pclk:
 879         clk_disable_unprepare(host->pclk);
 880         return ret;
 881 }
 882
 883 static void meson_mx_sdhc_remove(struct platform_device *pdev)
 884 {
 885         struct meson_mx_sdhc_host *host = platform_get_drvdata(pdev);
 886
 887         mmc_remove_host(host->mmc);
 888
 889         meson_mx_sdhc_disable_clks(host->mmc);
 890
 891         clk_disable_unprepare(host->pclk);
 892 }
 893
 894 static const struct meson_mx_sdhc_data meson_mx_sdhc_data_meson8 = {
 895         .init_hw                        = meson_mx_sdhc_init_hw_meson8,
 896         .set_pdma                       = meson_mx_sdhc_set_pdma_meson8,
 897         .wait_before_send               = meson_mx_sdhc_wait_before_send_meson8,
 898         .hardware_flush_all_cmds        = false,
 899 };
 900
 901 static const struct meson_mx_sdhc_data meson_mx_sdhc_data_meson8m2 = {
 902         .init_hw                        = meson_mx_sdhc_init_hw_meson8m2,
 903         .set_pdma                       = meson_mx_sdhc_set_pdma_meson8m2,
 904         .hardware_flush_all_cmds        = true,
 905 };
 906
 907 static const struct of_device_id meson_mx_sdhc_of_match[] = {
 908         {
 909                 .compatible = "amlogic,meson8-sdhc",
 910                 .data = &meson_mx_sdhc_data_meson8
 911         },
 912         {
 913                 .compatible = "amlogic,meson8b-sdhc",
 914                 .data = &meson_mx_sdhc_data_meson8
 915         },
 916         {
 917                 .compatible = "amlogic,meson8m2-sdhc",
 918                 .data = &meson_mx_sdhc_data_meson8m2
 919         },
 920         { /* sentinel */ }
 921 };
 922 MODULE_DEVICE_TABLE(of, meson_mx_sdhc_of_match);
 923
 924 static struct platform_driver meson_mx_sdhc_driver = {
 925         .probe   = meson_mx_sdhc_probe,
 926         .remove_new = meson_mx_sdhc_remove,
 927         .driver  = {
 928                 .name = "meson-mx-sdhc",
 929                 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 930                 .of_match_table = of_match_ptr(meson_mx_sdhc_of_match),
 931         },
 932 };
 933
 934 module_platform_driver(meson_mx_sdhc_driver);
 935
 936 MODULE_DESCRIPTION("Meson6, Meson8, Meson8b and Meson8m2 SDHC Host Driver");
 937 MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
 938 MODULE_LICENSE("GPL v2");