drivers/mmc/host/renesas_sdhi_sys_dmac.c

   1 /*
   2  * DMA support use of SYS DMAC with SDHI SD/SDIO controller
   3  *
   4  * Copyright (C) 2016-17 Renesas Electronics Corporation
   5  * Copyright (C) 2016-17 Sang Engineering, Wolfram Sang
   6  * Copyright (C) 2017 Horms Solutions, Simon Horman
   7  * Copyright (C) 2010-2011 Guennadi Liakhovetski
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/device.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/mfd/tmio.h>
  18 #include <linux/mmc/host.h>
  19 #include <linux/mod_devicetable.h>
  20 #include <linux/module.h>
  21 #include <linux/of_device.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sys_soc.h>
  25
  26 #include "renesas_sdhi.h"
  27 #include "tmio_mmc.h"
  28
  29 #define TMIO_MMC_MIN_DMA_LEN 8
  30
  31 static const struct renesas_sdhi_of_data of_default_cfg = {
  32         .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT,
  33 };
  34
  35 static const struct renesas_sdhi_of_data of_rz_compatible = {
  36         .tmio_flags     = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_32BIT_DATA_PORT |
  37                           TMIO_MMC_HAVE_CBSY,
  38         .tmio_ocr_mask  = MMC_VDD_32_33,
  39         .capabilities   = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
  40 };
  41
  42 static const struct renesas_sdhi_of_data of_rcar_gen1_compatible = {
  43         .tmio_flags     = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL,
  44         .capabilities   = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
  45         .capabilities2  = MMC_CAP2_NO_WRITE_PROTECT,
  46 };
  47
  48 /* Definitions for sampling clocks */
  49 static struct renesas_sdhi_scc rcar_gen2_scc_taps[] = {
  50         {
  51                 .clk_rate = 156000000,
  52                 .tap = 0x00000703,
  53         },
  54         {
  55                 .clk_rate = 0,
  56                 .tap = 0x00000300,
  57         },
  58 };
  59
  60 static const struct renesas_sdhi_of_data of_rcar_gen2_compatible = {
  61         .tmio_flags     = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL |
  62                           TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2,
  63         .capabilities   = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
  64                           MMC_CAP_CMD23,
  65         .capabilities2  = MMC_CAP2_NO_WRITE_PROTECT,
  66         .dma_buswidth   = DMA_SLAVE_BUSWIDTH_4_BYTES,
  67         .dma_rx_offset  = 0x2000,
  68         .scc_offset     = 0x0300,
  69         .taps           = rcar_gen2_scc_taps,
  70         .taps_num       = ARRAY_SIZE(rcar_gen2_scc_taps),
  71         .max_blk_count  = 0xffffffff,
  72 };
  73
  74 /* Definitions for sampling clocks */
  75 static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = {
  76         {
  77                 .clk_rate = 0,
  78                 .tap = 0x00000300,
  79         },
  80 };
  81
  82 static const struct renesas_sdhi_of_data of_rcar_r8a7795_compatible = {
  83         .tmio_flags     = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL |
  84                           TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2 |
  85                           TMIO_MMC_HAVE_4TAP_HS400,
  86         .capabilities   = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
  87                           MMC_CAP_CMD23,
  88         .capabilities2  = MMC_CAP2_NO_WRITE_PROTECT,
  89         .bus_shift      = 2,
  90         .scc_offset     = 0x1000,
  91         .taps           = rcar_gen3_scc_taps,
  92         .taps_num       = ARRAY_SIZE(rcar_gen3_scc_taps),
  93 };
  94
  95 static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = {
  96         .tmio_flags     = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL |
  97                           TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2,
  98         .capabilities   = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
  99                           MMC_CAP_CMD23,
 100         .capabilities2  = MMC_CAP2_NO_WRITE_PROTECT,
 101         .bus_shift      = 2,
 102         .scc_offset     = 0x1000,
 103         .taps           = rcar_gen3_scc_taps,
 104         .taps_num       = ARRAY_SIZE(rcar_gen3_scc_taps),
 105 };
 106
 107 static const struct of_device_id renesas_sdhi_sys_dmac_of_match[] = {
 108         { .compatible = "renesas,sdhi-sh73a0", .data = &of_default_cfg, },
 109         { .compatible = "renesas,sdhi-r8a73a4", .data = &of_default_cfg, },
 110         { .compatible = "renesas,sdhi-r8a7740", .data = &of_default_cfg, },
 111         { .compatible = "renesas,sdhi-r7s72100", .data = &of_rz_compatible, },
 112         { .compatible = "renesas,sdhi-r8a7778", .data = &of_rcar_gen1_compatible, },
 113         { .compatible = "renesas,sdhi-r8a7779", .data = &of_rcar_gen1_compatible, },
 114         { .compatible = "renesas,sdhi-r8a7743", .data = &of_rcar_gen2_compatible, },
 115         { .compatible = "renesas,sdhi-r8a7745", .data = &of_rcar_gen2_compatible, },
 116         { .compatible = "renesas,sdhi-r8a7790", .data = &of_rcar_gen2_compatible, },
 117         { .compatible = "renesas,sdhi-r8a7791", .data = &of_rcar_gen2_compatible, },
 118         { .compatible = "renesas,sdhi-r8a7792", .data = &of_rcar_gen2_compatible, },
 119         { .compatible = "renesas,sdhi-r8a7793", .data = &of_rcar_gen2_compatible, },
 120         { .compatible = "renesas,sdhi-r8a7794", .data = &of_rcar_gen2_compatible, },
 121         { .compatible = "renesas,sdhi-r8a7795", .data = &of_rcar_r8a7795_compatible, },
 122         { .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_r8a7795_compatible, },
 123         { .compatible = "renesas,rcar-gen1-sdhi", .data = &of_rcar_gen1_compatible, },
 124         { .compatible = "renesas,rcar-gen2-sdhi", .data = &of_rcar_gen2_compatible, },
 125         { .compatible = "renesas,rcar-gen3-sdhi", .data = &of_rcar_gen3_compatible, },
 126         { .compatible = "renesas,sdhi-shmobile" },
 127         {},
 128 };
 129 MODULE_DEVICE_TABLE(of, renesas_sdhi_sys_dmac_of_match);
 130
 131 static void renesas_sdhi_sys_dmac_enable_dma(struct tmio_mmc_host *host,
 132                                              bool enable)
 133 {
 134         struct renesas_sdhi *priv = host_to_priv(host);
 135
 136         if (!host->chan_tx || !host->chan_rx)
 137                 return;
 138
 139         if (priv->dma_priv.enable)
 140                 priv->dma_priv.enable(host, enable);
 141 }
 142
 143 static void renesas_sdhi_sys_dmac_abort_dma(struct tmio_mmc_host *host)
 144 {
 145         renesas_sdhi_sys_dmac_enable_dma(host, false);
 146
 147         if (host->chan_rx)
 148                 dmaengine_terminate_all(host->chan_rx);
 149         if (host->chan_tx)
 150                 dmaengine_terminate_all(host->chan_tx);
 151
 152         renesas_sdhi_sys_dmac_enable_dma(host, true);
 153 }
 154
 155 static void renesas_sdhi_sys_dmac_dataend_dma(struct tmio_mmc_host *host)
 156 {
 157         struct renesas_sdhi *priv = host_to_priv(host);
 158
 159         complete(&priv->dma_priv.dma_dataend);
 160 }
 161
 162 static void renesas_sdhi_sys_dmac_dma_callback(void *arg)
 163 {
 164         struct tmio_mmc_host *host = arg;
 165         struct renesas_sdhi *priv = host_to_priv(host);
 166
 167         spin_lock_irq(&host->lock);
 168
 169         if (!host->data)
 170                 goto out;
 171
 172         if (host->data->flags & MMC_DATA_READ)
 173                 dma_unmap_sg(host->chan_rx->device->dev,
 174                              host->sg_ptr, host->sg_len,
 175                              DMA_FROM_DEVICE);
 176         else
 177                 dma_unmap_sg(host->chan_tx->device->dev,
 178                              host->sg_ptr, host->sg_len,
 179                              DMA_TO_DEVICE);
 180
 181         spin_unlock_irq(&host->lock);
 182
 183         wait_for_completion(&priv->dma_priv.dma_dataend);
 184
 185         spin_lock_irq(&host->lock);
 186         tmio_mmc_do_data_irq(host);
 187 out:
 188         spin_unlock_irq(&host->lock);
 189 }
 190
 191 static void renesas_sdhi_sys_dmac_start_dma_rx(struct tmio_mmc_host *host)
 192 {
 193         struct renesas_sdhi *priv = host_to_priv(host);
 194         struct scatterlist *sg = host->sg_ptr, *sg_tmp;
 195         struct dma_async_tx_descriptor *desc = NULL;
 196         struct dma_chan *chan = host->chan_rx;
 197         dma_cookie_t cookie;
 198         int ret, i;
 199         bool aligned = true, multiple = true;
 200         unsigned int align = (1 << host->pdata->alignment_shift) - 1;
 201
 202         for_each_sg(sg, sg_tmp, host->sg_len, i) {
 203                 if (sg_tmp->offset & align)
 204                         aligned = false;
 205                 if (sg_tmp->length & align) {
 206                         multiple = false;
 207                         break;
 208                 }
 209         }
 210
 211         if ((!aligned && (host->sg_len > 1 || sg->length > PAGE_SIZE ||
 212                           (align & PAGE_MASK))) || !multiple) {
 213                 ret = -EINVAL;
 214                 goto pio;
 215         }
 216
 217         if (sg->length < TMIO_MMC_MIN_DMA_LEN) {
 218                 host->force_pio = true;
 219                 return;
 220         }
 221
 222         /* The only sg element can be unaligned, use our bounce buffer then */
 223         if (!aligned) {
 224                 sg_init_one(&host->bounce_sg, host->bounce_buf, sg->length);
 225                 host->sg_ptr = &host->bounce_sg;
 226                 sg = host->sg_ptr;
 227         }
 228
 229         ret = dma_map_sg(chan->device->dev, sg, host->sg_len, DMA_FROM_DEVICE);
 230         if (ret > 0)
 231                 desc = dmaengine_prep_slave_sg(chan, sg, ret, DMA_DEV_TO_MEM,
 232                                                DMA_CTRL_ACK);
 233
 234         if (desc) {
 235                 reinit_completion(&priv->dma_priv.dma_dataend);
 236                 desc->callback = renesas_sdhi_sys_dmac_dma_callback;
 237                 desc->callback_param = host;
 238
 239                 cookie = dmaengine_submit(desc);
 240                 if (cookie < 0) {
 241                         desc = NULL;
 242                         ret = cookie;
 243                 }
 244         }
 245 pio:
 246         if (!desc) {
 247                 /* DMA failed, fall back to PIO */
 248                 renesas_sdhi_sys_dmac_enable_dma(host, false);
 249                 if (ret >= 0)
 250                         ret = -EIO;
 251                 host->chan_rx = NULL;
 252                 dma_release_channel(chan);
 253                 /* Free the Tx channel too */
 254                 chan = host->chan_tx;
 255                 if (chan) {
 256                         host->chan_tx = NULL;
 257                         dma_release_channel(chan);
 258                 }
 259                 dev_warn(&host->pdev->dev,
 260                          "DMA failed: %d, falling back to PIO\n", ret);
 261         }
 262 }
 263
 264 static void renesas_sdhi_sys_dmac_start_dma_tx(struct tmio_mmc_host *host)
 265 {
 266         struct renesas_sdhi *priv = host_to_priv(host);
 267         struct scatterlist *sg = host->sg_ptr, *sg_tmp;
 268         struct dma_async_tx_descriptor *desc = NULL;
 269         struct dma_chan *chan = host->chan_tx;
 270         dma_cookie_t cookie;
 271         int ret, i;
 272         bool aligned = true, multiple = true;
 273         unsigned int align = (1 << host->pdata->alignment_shift) - 1;
 274
 275         for_each_sg(sg, sg_tmp, host->sg_len, i) {
 276                 if (sg_tmp->offset & align)
 277                         aligned = false;
 278                 if (sg_tmp->length & align) {
 279                         multiple = false;
 280                         break;
 281                 }
 282         }
 283
 284         if ((!aligned && (host->sg_len > 1 || sg->length > PAGE_SIZE ||
 285                           (align & PAGE_MASK))) || !multiple) {
 286                 ret = -EINVAL;
 287                 goto pio;
 288         }
 289
 290         if (sg->length < TMIO_MMC_MIN_DMA_LEN) {
 291                 host->force_pio = true;
 292                 return;
 293         }
 294
 295         /* The only sg element can be unaligned, use our bounce buffer then */
 296         if (!aligned) {
 297                 unsigned long flags;
 298                 void *sg_vaddr = tmio_mmc_kmap_atomic(sg, &flags);
 299
 300                 sg_init_one(&host->bounce_sg, host->bounce_buf, sg->length);
 301                 memcpy(host->bounce_buf, sg_vaddr, host->bounce_sg.length);
 302                 tmio_mmc_kunmap_atomic(sg, &flags, sg_vaddr);
 303                 host->sg_ptr = &host->bounce_sg;
 304                 sg = host->sg_ptr;
 305         }
 306
 307         ret = dma_map_sg(chan->device->dev, sg, host->sg_len, DMA_TO_DEVICE);
 308         if (ret > 0)
 309                 desc = dmaengine_prep_slave_sg(chan, sg, ret, DMA_MEM_TO_DEV,
 310                                                DMA_CTRL_ACK);
 311
 312         if (desc) {
 313                 reinit_completion(&priv->dma_priv.dma_dataend);
 314                 desc->callback = renesas_sdhi_sys_dmac_dma_callback;
 315                 desc->callback_param = host;
 316
 317                 cookie = dmaengine_submit(desc);
 318                 if (cookie < 0) {
 319                         desc = NULL;
 320                         ret = cookie;
 321                 }
 322         }
 323 pio:
 324         if (!desc) {
 325                 /* DMA failed, fall back to PIO */
 326                 renesas_sdhi_sys_dmac_enable_dma(host, false);
 327                 if (ret >= 0)
 328                         ret = -EIO;
 329                 host->chan_tx = NULL;
 330                 dma_release_channel(chan);
 331                 /* Free the Rx channel too */
 332                 chan = host->chan_rx;
 333                 if (chan) {
 334                         host->chan_rx = NULL;
 335                         dma_release_channel(chan);
 336                 }
 337                 dev_warn(&host->pdev->dev,
 338                          "DMA failed: %d, falling back to PIO\n", ret);
 339         }
 340 }
 341
 342 static void renesas_sdhi_sys_dmac_start_dma(struct tmio_mmc_host *host,
 343                                             struct mmc_data *data)
 344 {
 345         if (data->flags & MMC_DATA_READ) {
 346                 if (host->chan_rx)
 347                         renesas_sdhi_sys_dmac_start_dma_rx(host);
 348         } else {
 349                 if (host->chan_tx)
 350                         renesas_sdhi_sys_dmac_start_dma_tx(host);
 351         }
 352 }
 353
 354 static void renesas_sdhi_sys_dmac_issue_tasklet_fn(unsigned long priv)
 355 {
 356         struct tmio_mmc_host *host = (struct tmio_mmc_host *)priv;
 357         struct dma_chan *chan = NULL;
 358
 359         spin_lock_irq(&host->lock);
 360
 361         if (host->data) {
 362                 if (host->data->flags & MMC_DATA_READ)
 363                         chan = host->chan_rx;
 364                 else
 365                         chan = host->chan_tx;
 366         }
 367
 368         spin_unlock_irq(&host->lock);
 369
 370         tmio_mmc_enable_mmc_irqs(host, TMIO_STAT_DATAEND);
 371
 372         if (chan)
 373                 dma_async_issue_pending(chan);
 374 }
 375
 376 static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 377                                               struct tmio_mmc_data *pdata)
 378 {
 379         struct renesas_sdhi *priv = host_to_priv(host);
 380
 381         /* We can only either use DMA for both Tx and Rx or not use it at all */
 382         if (!host->pdev->dev.of_node &&
 383             (!pdata->chan_priv_tx || !pdata->chan_priv_rx))
 384                 return;
 385
 386         if (!host->chan_tx && !host->chan_rx) {
 387                 struct resource *res = platform_get_resource(host->pdev,
 388                                                              IORESOURCE_MEM, 0);
 389                 struct dma_slave_config cfg = {};
 390                 dma_cap_mask_t mask;
 391                 int ret;
 392
 393                 if (!res)
 394                         return;
 395
 396                 dma_cap_zero(mask);
 397                 dma_cap_set(DMA_SLAVE, mask);
 398
 399                 host->chan_tx = dma_request_slave_channel_compat(mask,
 400                                         priv->dma_priv.filter, pdata->chan_priv_tx,
 401                                         &host->pdev->dev, "tx");
 402                 dev_dbg(&host->pdev->dev, "%s: TX: got channel %p\n", __func__,
 403                         host->chan_tx);
 404
 405                 if (!host->chan_tx)
 406                         return;
 407
 408                 cfg.direction = DMA_MEM_TO_DEV;
 409                 cfg.dst_addr = res->start +
 410                         (CTL_SD_DATA_PORT << host->bus_shift);
 411                 cfg.dst_addr_width = priv->dma_priv.dma_buswidth;
 412                 if (!cfg.dst_addr_width)
 413                         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 414                 cfg.src_addr = 0;
 415                 ret = dmaengine_slave_config(host->chan_tx, &cfg);
 416                 if (ret < 0)
 417                         goto ecfgtx;
 418
 419                 host->chan_rx = dma_request_slave_channel_compat(mask,
 420                                         priv->dma_priv.filter, pdata->chan_priv_rx,
 421                                         &host->pdev->dev, "rx");
 422                 dev_dbg(&host->pdev->dev, "%s: RX: got channel %p\n", __func__,
 423                         host->chan_rx);
 424
 425                 if (!host->chan_rx)
 426                         goto ereqrx;
 427
 428                 cfg.direction = DMA_DEV_TO_MEM;
 429                 cfg.src_addr = cfg.dst_addr + host->pdata->dma_rx_offset;
 430                 cfg.src_addr_width = priv->dma_priv.dma_buswidth;
 431                 if (!cfg.src_addr_width)
 432                         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 433                 cfg.dst_addr = 0;
 434                 ret = dmaengine_slave_config(host->chan_rx, &cfg);
 435                 if (ret < 0)
 436                         goto ecfgrx;
 437
 438                 host->bounce_buf = (u8 *)__get_free_page(GFP_KERNEL | GFP_DMA);
 439                 if (!host->bounce_buf)
 440                         goto ebouncebuf;
 441
 442                 init_completion(&priv->dma_priv.dma_dataend);
 443                 tasklet_init(&host->dma_issue,
 444                              renesas_sdhi_sys_dmac_issue_tasklet_fn,
 445                              (unsigned long)host);
 446         }
 447
 448         renesas_sdhi_sys_dmac_enable_dma(host, true);
 449
 450         return;
 451
 452 ebouncebuf:
 453 ecfgrx:
 454         dma_release_channel(host->chan_rx);
 455         host->chan_rx = NULL;
 456 ereqrx:
 457 ecfgtx:
 458         dma_release_channel(host->chan_tx);
 459         host->chan_tx = NULL;
 460 }
 461
 462 static void renesas_sdhi_sys_dmac_release_dma(struct tmio_mmc_host *host)
 463 {
 464         if (host->chan_tx) {
 465                 struct dma_chan *chan = host->chan_tx;
 466
 467                 host->chan_tx = NULL;
 468                 dma_release_channel(chan);
 469         }
 470         if (host->chan_rx) {
 471                 struct dma_chan *chan = host->chan_rx;
 472
 473                 host->chan_rx = NULL;
 474                 dma_release_channel(chan);
 475         }
 476         if (host->bounce_buf) {
 477                 free_pages((unsigned long)host->bounce_buf, 0);
 478                 host->bounce_buf = NULL;
 479         }
 480 }
 481
 482 static const struct tmio_mmc_dma_ops renesas_sdhi_sys_dmac_dma_ops = {
 483         .start = renesas_sdhi_sys_dmac_start_dma,
 484         .enable = renesas_sdhi_sys_dmac_enable_dma,
 485         .request = renesas_sdhi_sys_dmac_request_dma,
 486         .release = renesas_sdhi_sys_dmac_release_dma,
 487         .abort = renesas_sdhi_sys_dmac_abort_dma,
 488         .dataend = renesas_sdhi_sys_dmac_dataend_dma,
 489 };
 490
 491 /*
 492  * Whitelist of specific R-Car Gen3 SoC ES versions to use this DMAC
 493  * implementation. Currently empty as all supported ES versions use
 494  * the internal DMAC.
 495  */
 496 static const struct soc_device_attribute gen3_soc_whitelist[] = {
 497         { /* sentinel */ }
 498 };
 499
 500 static int renesas_sdhi_sys_dmac_probe(struct platform_device *pdev)
 501 {
 502         if ((of_device_get_match_data(&pdev->dev) == &of_rcar_gen3_compatible ||
 503             of_device_get_match_data(&pdev->dev) == &of_rcar_r8a7795_compatible) &&
 504             !soc_device_match(gen3_soc_whitelist))
 505                 return -ENODEV;
 506
 507         return renesas_sdhi_probe(pdev, &renesas_sdhi_sys_dmac_dma_ops);
 508 }
 509
 510 static const struct dev_pm_ops renesas_sdhi_sys_dmac_dev_pm_ops = {
 511         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
 512                                 pm_runtime_force_resume)
 513         SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend,
 514                            tmio_mmc_host_runtime_resume,
 515                            NULL)
 516 };
 517
 518 static struct platform_driver renesas_sys_dmac_sdhi_driver = {
 519         .driver         = {
 520                 .name   = "sh_mobile_sdhi",
 521                 .pm     = &renesas_sdhi_sys_dmac_dev_pm_ops,
 522                 .of_match_table = renesas_sdhi_sys_dmac_of_match,
 523         },
 524         .probe          = renesas_sdhi_sys_dmac_probe,
 525         .remove         = renesas_sdhi_remove,
 526 };
 527
 528 module_platform_driver(renesas_sys_dmac_sdhi_driver);
 529
 530 MODULE_DESCRIPTION("Renesas SDHI driver");
 531 MODULE_AUTHOR("Magnus Damm");
 532 MODULE_LICENSE("GPL v2");
 533 MODULE_ALIAS("platform:sh_mobile_sdhi");