drivers/mtd/nand/raw/meson_nand.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
   2 /*
   3  * Amlogic Meson Nand Flash Controller Driver
   4  *
   5  * Copyright (c) 2018 Amlogic, inc.
   6  * Author: Liang Yang <liang.yang@amlogic.com>
   7  */
   8
   9 #include <linux/platform_device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/clk.h>
  13 #include <linux/mtd/rawnand.h>
  14 #include <linux/mtd/mtd.h>
  15 #include <linux/mfd/syscon.h>
  16 #include <linux/regmap.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/of.h>
  21 #include <linux/of_device.h>
  22 #include <linux/sched/task_stack.h>
  23
  24 #define NFC_REG_CMD             0x00
  25 #define NFC_CMD_IDLE            (0xc << 14)
  26 #define NFC_CMD_CLE             (0x5 << 14)
  27 #define NFC_CMD_ALE             (0x6 << 14)
  28 #define NFC_CMD_ADL             ((0 << 16) | (3 << 20))
  29 #define NFC_CMD_ADH             ((1 << 16) | (3 << 20))
  30 #define NFC_CMD_AIL             ((2 << 16) | (3 << 20))
  31 #define NFC_CMD_AIH             ((3 << 16) | (3 << 20))
  32 #define NFC_CMD_SEED            ((8 << 16) | (3 << 20))
  33 #define NFC_CMD_M2N             ((0 << 17) | (2 << 20))
  34 #define NFC_CMD_N2M             ((1 << 17) | (2 << 20))
  35 #define NFC_CMD_RB              BIT(20)
  36 #define NFC_CMD_SCRAMBLER_ENABLE        BIT(19)
  37 #define NFC_CMD_SCRAMBLER_DISABLE       0
  38 #define NFC_CMD_SHORTMODE_DISABLE       0
  39 #define NFC_CMD_RB_INT          BIT(14)
  40
  41 #define NFC_CMD_GET_SIZE(x)     (((x) >> 22) & GENMASK(4, 0))
  42
  43 #define NFC_REG_CFG             0x04
  44 #define NFC_REG_DADR            0x08
  45 #define NFC_REG_IADR            0x0c
  46 #define NFC_REG_BUF             0x10
  47 #define NFC_REG_INFO            0x14
  48 #define NFC_REG_DC              0x18
  49 #define NFC_REG_ADR             0x1c
  50 #define NFC_REG_DL              0x20
  51 #define NFC_REG_DH              0x24
  52 #define NFC_REG_CADR            0x28
  53 #define NFC_REG_SADR            0x2c
  54 #define NFC_REG_PINS            0x30
  55 #define NFC_REG_VER             0x38
  56
  57 #define NFC_RB_IRQ_EN           BIT(21)
  58
  59 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)       \
  60         (                                                               \
  61                 (cmd_dir)                       |                       \
  62                 (ran)                           |                       \
  63                 ((bch) << 14)                   |                       \
  64                 ((short_mode) << 13)            |                       \
  65                 (((page_size) & 0x7f) << 6)     |                       \
  66                 ((pages) & 0x3f)                                        \
  67         )
  68
  69 #define GENCMDDADDRL(adl, addr)         ((adl) | ((addr) & 0xffff))
  70 #define GENCMDDADDRH(adh, addr)         ((adh) | (((addr) >> 16) & 0xffff))
  71 #define GENCMDIADDRL(ail, addr)         ((ail) | ((addr) & 0xffff))
  72 #define GENCMDIADDRH(aih, addr)         ((aih) | (((addr) >> 16) & 0xffff))
  73
  74 #define DMA_DIR(dir)            ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
  75 #define DMA_ADDR_ALIGN          8
  76
  77 #define ECC_CHECK_RETURN_FF     (-1)
  78
  79 #define NAND_CE0                (0xe << 10)
  80 #define NAND_CE1                (0xd << 10)
  81
  82 #define DMA_BUSY_TIMEOUT        0x100000
  83 #define CMD_FIFO_EMPTY_TIMEOUT  1000
  84
  85 #define MAX_CE_NUM              2
  86
  87 /* eMMC clock register, misc control */
  88 #define CLK_SELECT_NAND         BIT(31)
  89
  90 #define NFC_CLK_CYCLE           6
  91
  92 /* nand flash controller delay 3 ns */
  93 #define NFC_DEFAULT_DELAY       3000
  94
  95 #define ROW_ADDER(page, index)  (((page) >> (8 * (index))) & 0xff)
  96 #define MAX_CYCLE_ADDRS         5
  97 #define DIRREAD                 1
  98 #define DIRWRITE                0
  99
 100 #define ECC_PARITY_BCH8_512B    14
 101 #define ECC_COMPLETE            BIT(31)
 102 #define ECC_ERR_CNT(x)          (((x) >> 24) & GENMASK(5, 0))
 103 #define ECC_ZERO_CNT(x)         (((x) >> 16) & GENMASK(5, 0))
 104 #define ECC_UNCORRECTABLE       0x3f
 105
 106 #define PER_INFO_BYTE           8
 107
 108 struct meson_nfc_nand_chip {
 109         struct list_head node;
 110         struct nand_chip nand;
 111         unsigned long clk_rate;
 112         unsigned long level1_divider;
 113         u32 bus_timing;
 114         u32 twb;
 115         u32 tadl;
 116         u32 tbers_max;
 117
 118         u32 bch_mode;
 119         u8 *data_buf;
 120         __le64 *info_buf;
 121         u32 nsels;
 122         u8 sels[];
 123 };
 124
 125 struct meson_nand_ecc {
 126         u32 bch;
 127         u32 strength;
 128 };
 129
 130 struct meson_nfc_data {
 131         const struct nand_ecc_caps *ecc_caps;
 132 };
 133
 134 struct meson_nfc_param {
 135         u32 chip_select;
 136         u32 rb_select;
 137 };
 138
 139 struct nand_rw_cmd {
 140         u32 cmd0;
 141         u32 addrs[MAX_CYCLE_ADDRS];
 142         u32 cmd1;
 143 };
 144
 145 struct nand_timing {
 146         u32 twb;
 147         u32 tadl;
 148         u32 tbers_max;
 149 };
 150
 151 struct meson_nfc {
 152         struct nand_controller controller;
 153         struct clk *core_clk;
 154         struct clk *device_clk;
 155         struct clk *phase_tx;
 156         struct clk *phase_rx;
 157
 158         unsigned long clk_rate;
 159         u32 bus_timing;
 160
 161         struct device *dev;
 162         void __iomem *reg_base;
 163         struct regmap *reg_clk;
 164         struct completion completion;
 165         struct list_head chips;
 166         const struct meson_nfc_data *data;
 167         struct meson_nfc_param param;
 168         struct nand_timing timing;
 169         union {
 170                 int cmd[32];
 171                 struct nand_rw_cmd rw;
 172         } cmdfifo;
 173
 174         dma_addr_t daddr;
 175         dma_addr_t iaddr;
 176         u32 info_bytes;
 177
 178         unsigned long assigned_cs;
 179 };
 180
 181 enum {
 182         NFC_ECC_BCH8_1K         = 2,
 183         NFC_ECC_BCH24_1K,
 184         NFC_ECC_BCH30_1K,
 185         NFC_ECC_BCH40_1K,
 186         NFC_ECC_BCH50_1K,
 187         NFC_ECC_BCH60_1K,
 188 };
 189
 190 #define MESON_ECC_DATA(b, s)    { .bch = (b),   .strength = (s)}
 191
 192 static struct meson_nand_ecc meson_ecc[] = {
 193         MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8),
 194         MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24),
 195         MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30),
 196         MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40),
 197         MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50),
 198         MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60),
 199 };
 200
 201 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
 202 {
 203         int ecc_bytes;
 204
 205         if (step_size == 512 && strength == 8)
 206                 return ECC_PARITY_BCH8_512B;
 207
 208         ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
 209         ecc_bytes = ALIGN(ecc_bytes, 2);
 210
 211         return ecc_bytes;
 212 }
 213
 214 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
 215                      meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
 216 NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps,
 217                      meson_nand_calc_ecc_bytes, 1024, 8);
 218
 219 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
 220 {
 221         return container_of(nand, struct meson_nfc_nand_chip, nand);
 222 }
 223
 224 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
 225 {
 226         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 227         struct meson_nfc *nfc = nand_get_controller_data(nand);
 228         int ret, value;
 229
 230         if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
 231                 return;
 232
 233         nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
 234         nfc->param.rb_select = nfc->param.chip_select;
 235         nfc->timing.twb = meson_chip->twb;
 236         nfc->timing.tadl = meson_chip->tadl;
 237         nfc->timing.tbers_max = meson_chip->tbers_max;
 238
 239         if (nfc->clk_rate != meson_chip->clk_rate) {
 240                 ret = clk_set_rate(nfc->device_clk, meson_chip->clk_rate);
 241                 if (ret) {
 242                         dev_err(nfc->dev, "failed to set clock rate\n");
 243                         return;
 244                 }
 245                 nfc->clk_rate = meson_chip->clk_rate;
 246         }
 247         if (nfc->bus_timing != meson_chip->bus_timing) {
 248                 value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
 249                 writel(value, nfc->reg_base + NFC_REG_CFG);
 250                 writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
 251                 nfc->bus_timing =  meson_chip->bus_timing;
 252         }
 253 }
 254
 255 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
 256 {
 257         writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
 258                nfc->reg_base + NFC_REG_CMD);
 259 }
 260
 261 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
 262 {
 263         writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
 264                nfc->reg_base + NFC_REG_CMD);
 265 }
 266
 267 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
 268                                  int scrambler)
 269 {
 270         struct mtd_info *mtd = nand_to_mtd(nand);
 271         struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
 272         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 273         u32 bch = meson_chip->bch_mode, cmd;
 274         int len = mtd->writesize, pagesize, pages;
 275
 276         pagesize = nand->ecc.size;
 277
 278         if (raw) {
 279                 len = mtd->writesize + mtd->oobsize;
 280                 cmd = (len & GENMASK(13, 0)) | scrambler | DMA_DIR(dir);
 281                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 282                 return;
 283         }
 284
 285         pages = len / nand->ecc.size;
 286
 287         cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
 288                        NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
 289
 290         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 291 }
 292
 293 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
 294 {
 295         /*
 296          * Insert two commands to make sure all valid commands are finished.
 297          *
 298          * The Nand flash controller is designed as two stages pipleline -
 299          *  a) fetch and b) excute.
 300          * There might be cases when the driver see command queue is empty,
 301          * but the Nand flash controller still has two commands buffered,
 302          * one is fetched into NFC request queue (ready to run), and another
 303          * is actively executing. So pushing 2 "IDLE" commands guarantees that
 304          * the pipeline is emptied.
 305          */
 306         meson_nfc_cmd_idle(nfc, 0);
 307         meson_nfc_cmd_idle(nfc, 0);
 308 }
 309
 310 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
 311                                      unsigned int timeout_ms)
 312 {
 313         u32 cmd_size = 0;
 314         int ret;
 315
 316         /* wait cmd fifo is empty */
 317         ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
 318                                          !NFC_CMD_GET_SIZE(cmd_size),
 319                                          10, timeout_ms * 1000);
 320         if (ret)
 321                 dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
 322
 323         return ret;
 324 }
 325
 326 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
 327 {
 328         meson_nfc_drain_cmd(nfc);
 329
 330         return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
 331 }
 332
 333 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
 334 {
 335         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 336         int len;
 337
 338         len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
 339
 340         return meson_chip->data_buf + len;
 341 }
 342
 343 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
 344 {
 345         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 346         int len, temp;
 347
 348         temp = nand->ecc.size + nand->ecc.bytes;
 349         len = (temp + 2) * i;
 350
 351         return meson_chip->data_buf + len;
 352 }
 353
 354 static void meson_nfc_get_data_oob(struct nand_chip *nand,
 355                                    u8 *buf, u8 *oobbuf)
 356 {
 357         int i, oob_len = 0;
 358         u8 *dsrc, *osrc;
 359
 360         oob_len = nand->ecc.bytes + 2;
 361         for (i = 0; i < nand->ecc.steps; i++) {
 362                 if (buf) {
 363                         dsrc = meson_nfc_data_ptr(nand, i);
 364                         memcpy(buf, dsrc, nand->ecc.size);
 365                         buf += nand->ecc.size;
 366                 }
 367                 osrc = meson_nfc_oob_ptr(nand, i);
 368                 memcpy(oobbuf, osrc, oob_len);
 369                 oobbuf += oob_len;
 370         }
 371 }
 372
 373 static void meson_nfc_set_data_oob(struct nand_chip *nand,
 374                                    const u8 *buf, u8 *oobbuf)
 375 {
 376         int i, oob_len = 0;
 377         u8 *dsrc, *osrc;
 378
 379         oob_len = nand->ecc.bytes + 2;
 380         for (i = 0; i < nand->ecc.steps; i++) {
 381                 if (buf) {
 382                         dsrc = meson_nfc_data_ptr(nand, i);
 383                         memcpy(dsrc, buf, nand->ecc.size);
 384                         buf += nand->ecc.size;
 385                 }
 386                 osrc = meson_nfc_oob_ptr(nand, i);
 387                 memcpy(osrc, oobbuf, oob_len);
 388                 oobbuf += oob_len;
 389         }
 390 }
 391
 392 static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms)
 393 {
 394         u32 cmd, cfg;
 395         int ret = 0;
 396
 397         meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 398         meson_nfc_drain_cmd(nfc);
 399         meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 400
 401         cfg = readl(nfc->reg_base + NFC_REG_CFG);
 402         cfg |= NFC_RB_IRQ_EN;
 403         writel(cfg, nfc->reg_base + NFC_REG_CFG);
 404
 405         reinit_completion(&nfc->completion);
 406
 407         /* use the max erase time as the maximum clock for waiting R/B */
 408         cmd = NFC_CMD_RB | NFC_CMD_RB_INT
 409                 | nfc->param.chip_select | nfc->timing.tbers_max;
 410         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 411
 412         ret = wait_for_completion_timeout(&nfc->completion,
 413                                           msecs_to_jiffies(timeout_ms));
 414         if (ret == 0)
 415                 ret = -1;
 416
 417         return ret;
 418 }
 419
 420 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
 421 {
 422         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 423         __le64 *info;
 424         int i, count;
 425
 426         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 427                 info = &meson_chip->info_buf[i];
 428                 *info |= oob_buf[count];
 429                 *info |= oob_buf[count + 1] << 8;
 430         }
 431 }
 432
 433 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
 434 {
 435         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 436         __le64 *info;
 437         int i, count;
 438
 439         for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 440                 info = &meson_chip->info_buf[i];
 441                 oob_buf[count] = *info;
 442                 oob_buf[count + 1] = *info >> 8;
 443         }
 444 }
 445
 446 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
 447                                  u64 *correct_bitmap)
 448 {
 449         struct mtd_info *mtd = nand_to_mtd(nand);
 450         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 451         __le64 *info;
 452         int ret = 0, i;
 453
 454         for (i = 0; i < nand->ecc.steps; i++) {
 455                 info = &meson_chip->info_buf[i];
 456                 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
 457                         mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
 458                         *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
 459                         *correct_bitmap |= BIT_ULL(i);
 460                         continue;
 461                 }
 462                 if ((nand->options & NAND_NEED_SCRAMBLING) &&
 463                     ECC_ZERO_CNT(*info) < nand->ecc.strength) {
 464                         mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
 465                         *bitflips = max_t(u32, *bitflips,
 466                                           ECC_ZERO_CNT(*info));
 467                         ret = ECC_CHECK_RETURN_FF;
 468                 } else {
 469                         ret = -EBADMSG;
 470                 }
 471         }
 472         return ret;
 473 }
 474
 475 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
 476                                       int datalen, void *infobuf, int infolen,
 477                                       enum dma_data_direction dir)
 478 {
 479         struct meson_nfc *nfc = nand_get_controller_data(nand);
 480         u32 cmd;
 481         int ret = 0;
 482
 483         nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
 484         ret = dma_mapping_error(nfc->dev, nfc->daddr);
 485         if (ret) {
 486                 dev_err(nfc->dev, "DMA mapping error\n");
 487                 return ret;
 488         }
 489         cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
 490         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 491
 492         cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
 493         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 494
 495         if (infobuf) {
 496                 nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
 497                 ret = dma_mapping_error(nfc->dev, nfc->iaddr);
 498                 if (ret) {
 499                         dev_err(nfc->dev, "DMA mapping error\n");
 500                         dma_unmap_single(nfc->dev,
 501                                          nfc->daddr, datalen, dir);
 502                         return ret;
 503                 }
 504                 nfc->info_bytes = infolen;
 505                 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
 506                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 507
 508                 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
 509                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 510         }
 511
 512         return ret;
 513 }
 514
 515 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
 516                                          int datalen, int infolen,
 517                                          enum dma_data_direction dir)
 518 {
 519         struct meson_nfc *nfc = nand_get_controller_data(nand);
 520
 521         dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
 522         if (infolen) {
 523                 dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
 524                 nfc->info_bytes = 0;
 525         }
 526 }
 527
 528 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
 529 {
 530         struct meson_nfc *nfc = nand_get_controller_data(nand);
 531         int ret = 0;
 532         u32 cmd;
 533         u8 *info;
 534
 535         info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
 536         if (!info)
 537                 return -ENOMEM;
 538
 539         ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
 540                                          PER_INFO_BYTE, DMA_FROM_DEVICE);
 541         if (ret)
 542                 goto out;
 543
 544         cmd = NFC_CMD_N2M | (len & GENMASK(13, 0));
 545         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 546
 547         meson_nfc_drain_cmd(nfc);
 548         meson_nfc_wait_cmd_finish(nfc, 1000);
 549         meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
 550
 551 out:
 552         kfree(info);
 553
 554         return ret;
 555 }
 556
 557 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
 558 {
 559         struct meson_nfc *nfc = nand_get_controller_data(nand);
 560         int ret = 0;
 561         u32 cmd;
 562
 563         ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
 564                                          0, DMA_TO_DEVICE);
 565         if (ret)
 566                 return ret;
 567
 568         cmd = NFC_CMD_M2N | (len & GENMASK(13, 0));
 569         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 570
 571         meson_nfc_drain_cmd(nfc);
 572         meson_nfc_wait_cmd_finish(nfc, 1000);
 573         meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
 574
 575         return ret;
 576 }
 577
 578 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
 579                                                 int page, bool in)
 580 {
 581         const struct nand_sdr_timings *sdr =
 582                 nand_get_sdr_timings(nand_get_interface_config(nand));
 583         struct mtd_info *mtd = nand_to_mtd(nand);
 584         struct meson_nfc *nfc = nand_get_controller_data(nand);
 585         u32 *addrs = nfc->cmdfifo.rw.addrs;
 586         u32 cs = nfc->param.chip_select;
 587         u32 cmd0, cmd_num, row_start;
 588         int ret = 0, i;
 589
 590         cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
 591
 592         cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
 593         nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
 594
 595         addrs[0] = cs | NFC_CMD_ALE | 0;
 596         if (mtd->writesize <= 512) {
 597                 cmd_num--;
 598                 row_start = 1;
 599         } else {
 600                 addrs[1] = cs | NFC_CMD_ALE | 0;
 601                 row_start = 2;
 602         }
 603
 604         addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
 605         addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
 606
 607         if (nand->options & NAND_ROW_ADDR_3)
 608                 addrs[row_start + 2] =
 609                         cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
 610         else
 611                 cmd_num--;
 612
 613         /* subtract cmd1 */
 614         cmd_num--;
 615
 616         for (i = 0; i < cmd_num; i++)
 617                 writel_relaxed(nfc->cmdfifo.cmd[i],
 618                                nfc->reg_base + NFC_REG_CMD);
 619
 620         if (in) {
 621                 nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
 622                 writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
 623                 meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max));
 624         } else {
 625                 meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
 626         }
 627
 628         return ret;
 629 }
 630
 631 static int meson_nfc_write_page_sub(struct nand_chip *nand,
 632                                     int page, int raw)
 633 {
 634         const struct nand_sdr_timings *sdr =
 635                 nand_get_sdr_timings(nand_get_interface_config(nand));
 636         struct mtd_info *mtd = nand_to_mtd(nand);
 637         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 638         struct meson_nfc *nfc = nand_get_controller_data(nand);
 639         int data_len, info_len;
 640         u32 cmd;
 641         int ret;
 642
 643         meson_nfc_select_chip(nand, nand->cur_cs);
 644
 645         data_len =  mtd->writesize + mtd->oobsize;
 646         info_len = nand->ecc.steps * PER_INFO_BYTE;
 647
 648         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
 649         if (ret)
 650                 return ret;
 651
 652         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 653                                          data_len, meson_chip->info_buf,
 654                                          info_len, DMA_TO_DEVICE);
 655         if (ret)
 656                 return ret;
 657
 658         if (nand->options & NAND_NEED_SCRAMBLING) {
 659                 meson_nfc_cmd_seed(nfc, page);
 660                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 661                                      NFC_CMD_SCRAMBLER_ENABLE);
 662         } else {
 663                 meson_nfc_cmd_access(nand, raw, DIRWRITE,
 664                                      NFC_CMD_SCRAMBLER_DISABLE);
 665         }
 666
 667         cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
 668         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 669         meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max));
 670
 671         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
 672
 673         return ret;
 674 }
 675
 676 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
 677                                     int oob_required, int page)
 678 {
 679         u8 *oob_buf = nand->oob_poi;
 680
 681         meson_nfc_set_data_oob(nand, buf, oob_buf);
 682
 683         return meson_nfc_write_page_sub(nand, page, 1);
 684 }
 685
 686 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
 687                                       const u8 *buf, int oob_required, int page)
 688 {
 689         struct mtd_info *mtd = nand_to_mtd(nand);
 690         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 691         u8 *oob_buf = nand->oob_poi;
 692
 693         memcpy(meson_chip->data_buf, buf, mtd->writesize);
 694         memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
 695         meson_nfc_set_user_byte(nand, oob_buf);
 696
 697         return meson_nfc_write_page_sub(nand, page, 0);
 698 }
 699
 700 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
 701                                             struct nand_chip *nand, int raw)
 702 {
 703         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 704         __le64 *info;
 705         u32 neccpages;
 706         int ret;
 707
 708         neccpages = raw ? 1 : nand->ecc.steps;
 709         info = &meson_chip->info_buf[neccpages - 1];
 710         do {
 711                 usleep_range(10, 15);
 712                 /* info is updated by nfc dma engine*/
 713                 smp_rmb();
 714                 dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
 715                                         DMA_FROM_DEVICE);
 716                 ret = *info & ECC_COMPLETE;
 717         } while (!ret);
 718 }
 719
 720 static int meson_nfc_read_page_sub(struct nand_chip *nand,
 721                                    int page, int raw)
 722 {
 723         struct mtd_info *mtd = nand_to_mtd(nand);
 724         struct meson_nfc *nfc = nand_get_controller_data(nand);
 725         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 726         int data_len, info_len;
 727         int ret;
 728
 729         meson_nfc_select_chip(nand, nand->cur_cs);
 730
 731         data_len =  mtd->writesize + mtd->oobsize;
 732         info_len = nand->ecc.steps * PER_INFO_BYTE;
 733
 734         ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
 735         if (ret)
 736                 return ret;
 737
 738         ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 739                                          data_len, meson_chip->info_buf,
 740                                          info_len, DMA_FROM_DEVICE);
 741         if (ret)
 742                 return ret;
 743
 744         if (nand->options & NAND_NEED_SCRAMBLING) {
 745                 meson_nfc_cmd_seed(nfc, page);
 746                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 747                                      NFC_CMD_SCRAMBLER_ENABLE);
 748         } else {
 749                 meson_nfc_cmd_access(nand, raw, DIRREAD,
 750                                      NFC_CMD_SCRAMBLER_DISABLE);
 751         }
 752
 753         ret = meson_nfc_wait_dma_finish(nfc);
 754         meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
 755
 756         meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
 757
 758         return ret;
 759 }
 760
 761 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
 762                                    int oob_required, int page)
 763 {
 764         u8 *oob_buf = nand->oob_poi;
 765         int ret;
 766
 767         ret = meson_nfc_read_page_sub(nand, page, 1);
 768         if (ret)
 769                 return ret;
 770
 771         meson_nfc_get_data_oob(nand, buf, oob_buf);
 772
 773         return 0;
 774 }
 775
 776 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
 777                                      int oob_required, int page)
 778 {
 779         struct mtd_info *mtd = nand_to_mtd(nand);
 780         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 781         struct nand_ecc_ctrl *ecc = &nand->ecc;
 782         u64 correct_bitmap = 0;
 783         u32 bitflips = 0;
 784         u8 *oob_buf = nand->oob_poi;
 785         int ret, i;
 786
 787         ret = meson_nfc_read_page_sub(nand, page, 0);
 788         if (ret)
 789                 return ret;
 790
 791         meson_nfc_get_user_byte(nand, oob_buf);
 792         ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
 793         if (ret == ECC_CHECK_RETURN_FF) {
 794                 if (buf)
 795                         memset(buf, 0xff, mtd->writesize);
 796                 memset(oob_buf, 0xff, mtd->oobsize);
 797         } else if (ret < 0) {
 798                 if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
 799                         mtd->ecc_stats.failed++;
 800                         return bitflips;
 801                 }
 802                 ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
 803                 if (ret)
 804                         return ret;
 805
 806                 for (i = 0; i < nand->ecc.steps ; i++) {
 807                         u8 *data = buf + i * ecc->size;
 808                         u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
 809
 810                         if (correct_bitmap & BIT_ULL(i))
 811                                 continue;
 812                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
 813                                                           oob, ecc->bytes + 2,
 814                                                           NULL, 0,
 815                                                           ecc->strength);
 816                         if (ret < 0) {
 817                                 mtd->ecc_stats.failed++;
 818                         } else {
 819                                 mtd->ecc_stats.corrected += ret;
 820                                 bitflips =  max_t(u32, bitflips, ret);
 821                         }
 822                 }
 823         } else if (buf && buf != meson_chip->data_buf) {
 824                 memcpy(buf, meson_chip->data_buf, mtd->writesize);
 825         }
 826
 827         return bitflips;
 828 }
 829
 830 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
 831 {
 832         return meson_nfc_read_page_raw(nand, NULL, 1, page);
 833 }
 834
 835 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
 836 {
 837         return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
 838 }
 839
 840 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
 841 {
 842         if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
 843                 return false;
 844
 845         if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
 846                 return true;
 847         return false;
 848 }
 849
 850 static void *
 851 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
 852 {
 853         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
 854                 return NULL;
 855
 856         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
 857                 return instr->ctx.data.buf.in;
 858
 859         return kzalloc(instr->ctx.data.len, GFP_KERNEL);
 860 }
 861
 862 static void
 863 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
 864                                      void *buf)
 865 {
 866         if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
 867             WARN_ON(!buf))
 868                 return;
 869
 870         if (buf == instr->ctx.data.buf.in)
 871                 return;
 872
 873         memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
 874         kfree(buf);
 875 }
 876
 877 static void *
 878 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
 879 {
 880         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
 881                 return NULL;
 882
 883         if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
 884                 return (void *)instr->ctx.data.buf.out;
 885
 886         return kmemdup(instr->ctx.data.buf.out,
 887                        instr->ctx.data.len, GFP_KERNEL);
 888 }
 889
 890 static void
 891 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
 892                                       const void *buf)
 893 {
 894         if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
 895             WARN_ON(!buf))
 896                 return;
 897
 898         if (buf != instr->ctx.data.buf.out)
 899                 kfree(buf);
 900 }
 901
 902 static int meson_nfc_exec_op(struct nand_chip *nand,
 903                              const struct nand_operation *op, bool check_only)
 904 {
 905         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 906         struct meson_nfc *nfc = nand_get_controller_data(nand);
 907         const struct nand_op_instr *instr = NULL;
 908         void *buf;
 909         u32 op_id, delay_idle, cmd;
 910         int i;
 911
 912         if (check_only)
 913                 return 0;
 914
 915         meson_nfc_select_chip(nand, op->cs);
 916         for (op_id = 0; op_id < op->ninstrs; op_id++) {
 917                 instr = &op->instrs[op_id];
 918                 delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
 919                                           meson_chip->level1_divider *
 920                                           NFC_CLK_CYCLE);
 921                 switch (instr->type) {
 922                 case NAND_OP_CMD_INSTR:
 923                         cmd = nfc->param.chip_select | NFC_CMD_CLE;
 924                         cmd |= instr->ctx.cmd.opcode & 0xff;
 925                         writel(cmd, nfc->reg_base + NFC_REG_CMD);
 926                         meson_nfc_cmd_idle(nfc, delay_idle);
 927                         break;
 928
 929                 case NAND_OP_ADDR_INSTR:
 930                         for (i = 0; i < instr->ctx.addr.naddrs; i++) {
 931                                 cmd = nfc->param.chip_select | NFC_CMD_ALE;
 932                                 cmd |= instr->ctx.addr.addrs[i] & 0xff;
 933                                 writel(cmd, nfc->reg_base + NFC_REG_CMD);
 934                         }
 935                         meson_nfc_cmd_idle(nfc, delay_idle);
 936                         break;
 937
 938                 case NAND_OP_DATA_IN_INSTR:
 939                         buf = meson_nand_op_get_dma_safe_input_buf(instr);
 940                         if (!buf)
 941                                 return -ENOMEM;
 942                         meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
 943                         meson_nand_op_put_dma_safe_input_buf(instr, buf);
 944                         break;
 945
 946                 case NAND_OP_DATA_OUT_INSTR:
 947                         buf = meson_nand_op_get_dma_safe_output_buf(instr);
 948                         if (!buf)
 949                                 return -ENOMEM;
 950                         meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
 951                         meson_nand_op_put_dma_safe_output_buf(instr, buf);
 952                         break;
 953
 954                 case NAND_OP_WAITRDY_INSTR:
 955                         meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms);
 956                         if (instr->delay_ns)
 957                                 meson_nfc_cmd_idle(nfc, delay_idle);
 958                         break;
 959                 }
 960         }
 961         meson_nfc_wait_cmd_finish(nfc, 1000);
 962         return 0;
 963 }
 964
 965 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
 966                                struct mtd_oob_region *oobregion)
 967 {
 968         struct nand_chip *nand = mtd_to_nand(mtd);
 969
 970         if (section >= nand->ecc.steps)
 971                 return -ERANGE;
 972
 973         oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
 974         oobregion->length = nand->ecc.bytes;
 975
 976         return 0;
 977 }
 978
 979 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
 980                                 struct mtd_oob_region *oobregion)
 981 {
 982         struct nand_chip *nand = mtd_to_nand(mtd);
 983
 984         if (section >= nand->ecc.steps)
 985                 return -ERANGE;
 986
 987         oobregion->offset = section * (2 + nand->ecc.bytes);
 988         oobregion->length = 2;
 989
 990         return 0;
 991 }
 992
 993 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
 994         .ecc = meson_ooblayout_ecc,
 995         .free = meson_ooblayout_free,
 996 };
 997
 998 static int meson_nfc_clk_init(struct meson_nfc *nfc)
 999 {
1000         int ret;
1001
1002         /* request core clock */
1003         nfc->core_clk = devm_clk_get(nfc->dev, "core");
1004         if (IS_ERR(nfc->core_clk)) {
1005                 dev_err(nfc->dev, "failed to get core clock\n");
1006                 return PTR_ERR(nfc->core_clk);
1007         }
1008
1009         nfc->device_clk = devm_clk_get(nfc->dev, "device");
1010         if (IS_ERR(nfc->device_clk)) {
1011                 dev_err(nfc->dev, "failed to get device clock\n");
1012                 return PTR_ERR(nfc->device_clk);
1013         }
1014
1015         nfc->phase_tx = devm_clk_get(nfc->dev, "tx");
1016         if (IS_ERR(nfc->phase_tx)) {
1017                 dev_err(nfc->dev, "failed to get TX clk\n");
1018                 return PTR_ERR(nfc->phase_tx);
1019         }
1020
1021         nfc->phase_rx = devm_clk_get(nfc->dev, "rx");
1022         if (IS_ERR(nfc->phase_rx)) {
1023                 dev_err(nfc->dev, "failed to get RX clk\n");
1024                 return PTR_ERR(nfc->phase_rx);
1025         }
1026
1027         /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1028         regmap_update_bits(nfc->reg_clk,
1029                            0, CLK_SELECT_NAND, CLK_SELECT_NAND);
1030
1031         ret = clk_prepare_enable(nfc->core_clk);
1032         if (ret) {
1033                 dev_err(nfc->dev, "failed to enable core clock\n");
1034                 return ret;
1035         }
1036
1037         ret = clk_prepare_enable(nfc->device_clk);
1038         if (ret) {
1039                 dev_err(nfc->dev, "failed to enable device clock\n");
1040                 goto err_device_clk;
1041         }
1042
1043         ret = clk_prepare_enable(nfc->phase_tx);
1044         if (ret) {
1045                 dev_err(nfc->dev, "failed to enable TX clock\n");
1046                 goto err_phase_tx;
1047         }
1048
1049         ret = clk_prepare_enable(nfc->phase_rx);
1050         if (ret) {
1051                 dev_err(nfc->dev, "failed to enable RX clock\n");
1052                 goto err_phase_rx;
1053         }
1054
1055         ret = clk_set_rate(nfc->device_clk, 24000000);
1056         if (ret)
1057                 goto err_disable_rx;
1058
1059         return 0;
1060
1061 err_disable_rx:
1062         clk_disable_unprepare(nfc->phase_rx);
1063 err_phase_rx:
1064         clk_disable_unprepare(nfc->phase_tx);
1065 err_phase_tx:
1066         clk_disable_unprepare(nfc->device_clk);
1067 err_device_clk:
1068         clk_disable_unprepare(nfc->core_clk);
1069         return ret;
1070 }
1071
1072 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1073 {
1074         clk_disable_unprepare(nfc->phase_rx);
1075         clk_disable_unprepare(nfc->phase_tx);
1076         clk_disable_unprepare(nfc->device_clk);
1077         clk_disable_unprepare(nfc->core_clk);
1078 }
1079
1080 static void meson_nfc_free_buffer(struct nand_chip *nand)
1081 {
1082         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1083
1084         kfree(meson_chip->info_buf);
1085         kfree(meson_chip->data_buf);
1086 }
1087
1088 static int meson_chip_buffer_init(struct nand_chip *nand)
1089 {
1090         struct mtd_info *mtd = nand_to_mtd(nand);
1091         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1092         u32 page_bytes, info_bytes, nsectors;
1093
1094         nsectors = mtd->writesize / nand->ecc.size;
1095
1096         page_bytes =  mtd->writesize + mtd->oobsize;
1097         info_bytes = nsectors * PER_INFO_BYTE;
1098
1099         meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1100         if (!meson_chip->data_buf)
1101                 return -ENOMEM;
1102
1103         meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1104         if (!meson_chip->info_buf) {
1105                 kfree(meson_chip->data_buf);
1106                 return -ENOMEM;
1107         }
1108
1109         return 0;
1110 }
1111
1112 static
1113 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1114                               const struct nand_interface_config *conf)
1115 {
1116         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1117         const struct nand_sdr_timings *timings;
1118         u32 div, bt_min, bt_max, tbers_clocks;
1119
1120         timings = nand_get_sdr_timings(conf);
1121         if (IS_ERR(timings))
1122                 return -ENOTSUPP;
1123
1124         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1125                 return 0;
1126
1127         div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1128         bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1129         bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1130                   timings->tRC_min / 2) / div;
1131
1132         meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1133                                        div * NFC_CLK_CYCLE);
1134         meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1135                                         div * NFC_CLK_CYCLE);
1136         tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1137                                         div * NFC_CLK_CYCLE);
1138         meson_chip->tbers_max = ilog2(tbers_clocks);
1139         if (!is_power_of_2(tbers_clocks))
1140                 meson_chip->tbers_max++;
1141
1142         bt_min = DIV_ROUND_UP(bt_min, 1000);
1143         bt_max = DIV_ROUND_UP(bt_max, 1000);
1144
1145         if (bt_max < bt_min)
1146                 return -EINVAL;
1147
1148         meson_chip->level1_divider = div;
1149         meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1150         meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1151
1152         return 0;
1153 }
1154
1155 static int meson_nand_bch_mode(struct nand_chip *nand)
1156 {
1157         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1158         int i;
1159
1160         if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1161                 return -EINVAL;
1162
1163         for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1164                 if (meson_ecc[i].strength == nand->ecc.strength) {
1165                         meson_chip->bch_mode = meson_ecc[i].bch;
1166                         return 0;
1167                 }
1168         }
1169
1170         return -EINVAL;
1171 }
1172
1173 static void meson_nand_detach_chip(struct nand_chip *nand)
1174 {
1175         meson_nfc_free_buffer(nand);
1176 }
1177
1178 static int meson_nand_attach_chip(struct nand_chip *nand)
1179 {
1180         struct meson_nfc *nfc = nand_get_controller_data(nand);
1181         struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1182         struct mtd_info *mtd = nand_to_mtd(nand);
1183         int ret;
1184
1185         if (!mtd->name) {
1186                 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1187                                            "%s:nand%d",
1188                                            dev_name(nfc->dev),
1189                                            meson_chip->sels[0]);
1190                 if (!mtd->name)
1191                         return -ENOMEM;
1192         }
1193
1194         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1195                 nand->bbt_options |= NAND_BBT_NO_OOB;
1196
1197         nand->options |= NAND_NO_SUBPAGE_WRITE;
1198
1199         ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1200                                    mtd->oobsize - 2);
1201         if (ret) {
1202                 dev_err(nfc->dev, "failed to ECC init\n");
1203                 return -EINVAL;
1204         }
1205
1206         mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1207
1208         ret = meson_nand_bch_mode(nand);
1209         if (ret)
1210                 return -EINVAL;
1211
1212         nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1213         nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1214         nand->ecc.write_page = meson_nfc_write_page_hwecc;
1215         nand->ecc.write_oob_raw = nand_write_oob_std;
1216         nand->ecc.write_oob = nand_write_oob_std;
1217
1218         nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1219         nand->ecc.read_page = meson_nfc_read_page_hwecc;
1220         nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1221         nand->ecc.read_oob = meson_nfc_read_oob;
1222
1223         if (nand->options & NAND_BUSWIDTH_16) {
1224                 dev_err(nfc->dev, "16bits bus width not supported");
1225                 return -EINVAL;
1226         }
1227         ret = meson_chip_buffer_init(nand);
1228         if (ret)
1229                 return -ENOMEM;
1230
1231         return ret;
1232 }
1233
1234 static const struct nand_controller_ops meson_nand_controller_ops = {
1235         .attach_chip = meson_nand_attach_chip,
1236         .detach_chip = meson_nand_detach_chip,
1237         .setup_interface = meson_nfc_setup_interface,
1238         .exec_op = meson_nfc_exec_op,
1239 };
1240
1241 static int
1242 meson_nfc_nand_chip_init(struct device *dev,
1243                          struct meson_nfc *nfc, struct device_node *np)
1244 {
1245         struct meson_nfc_nand_chip *meson_chip;
1246         struct nand_chip *nand;
1247         struct mtd_info *mtd;
1248         int ret, i;
1249         u32 tmp, nsels;
1250
1251         nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1252         if (!nsels || nsels > MAX_CE_NUM) {
1253                 dev_err(dev, "invalid register property size\n");
1254                 return -EINVAL;
1255         }
1256
1257         meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1258                                   GFP_KERNEL);
1259         if (!meson_chip)
1260                 return -ENOMEM;
1261
1262         meson_chip->nsels = nsels;
1263
1264         for (i = 0; i < nsels; i++) {
1265                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1266                 if (ret) {
1267                         dev_err(dev, "could not retrieve register property: %d\n",
1268                                 ret);
1269                         return ret;
1270                 }
1271
1272                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1273                         dev_err(dev, "CS %d already assigned\n", tmp);
1274                         return -EINVAL;
1275                 }
1276         }
1277
1278         nand = &meson_chip->nand;
1279         nand->controller = &nfc->controller;
1280         nand->controller->ops = &meson_nand_controller_ops;
1281         nand_set_flash_node(nand, np);
1282         nand_set_controller_data(nand, nfc);
1283
1284         nand->options |= NAND_USES_DMA;
1285         mtd = nand_to_mtd(nand);
1286         mtd->owner = THIS_MODULE;
1287         mtd->dev.parent = dev;
1288
1289         ret = nand_scan(nand, nsels);
1290         if (ret)
1291                 return ret;
1292
1293         ret = mtd_device_register(mtd, NULL, 0);
1294         if (ret) {
1295                 dev_err(dev, "failed to register MTD device: %d\n", ret);
1296                 nand_cleanup(nand);
1297                 return ret;
1298         }
1299
1300         list_add_tail(&meson_chip->node, &nfc->chips);
1301
1302         return 0;
1303 }
1304
1305 static int meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1306 {
1307         struct meson_nfc_nand_chip *meson_chip;
1308         struct mtd_info *mtd;
1309         int ret;
1310
1311         while (!list_empty(&nfc->chips)) {
1312                 meson_chip = list_first_entry(&nfc->chips,
1313                                               struct meson_nfc_nand_chip, node);
1314                 mtd = nand_to_mtd(&meson_chip->nand);
1315                 ret = mtd_device_unregister(mtd);
1316                 if (ret)
1317                         return ret;
1318
1319                 nand_cleanup(&meson_chip->nand);
1320                 list_del(&meson_chip->node);
1321         }
1322
1323         return 0;
1324 }
1325
1326 static int meson_nfc_nand_chips_init(struct device *dev,
1327                                      struct meson_nfc *nfc)
1328 {
1329         struct device_node *np = dev->of_node;
1330         struct device_node *nand_np;
1331         int ret;
1332
1333         for_each_child_of_node(np, nand_np) {
1334                 ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1335                 if (ret) {
1336                         meson_nfc_nand_chip_cleanup(nfc);
1337                         of_node_put(nand_np);
1338                         return ret;
1339                 }
1340         }
1341
1342         return 0;
1343 }
1344
1345 static irqreturn_t meson_nfc_irq(int irq, void *id)
1346 {
1347         struct meson_nfc *nfc = id;
1348         u32 cfg;
1349
1350         cfg = readl(nfc->reg_base + NFC_REG_CFG);
1351         if (!(cfg & NFC_RB_IRQ_EN))
1352                 return IRQ_NONE;
1353
1354         cfg &= ~(NFC_RB_IRQ_EN);
1355         writel(cfg, nfc->reg_base + NFC_REG_CFG);
1356
1357         complete(&nfc->completion);
1358         return IRQ_HANDLED;
1359 }
1360
1361 static const struct meson_nfc_data meson_gxl_data = {
1362         .ecc_caps = &meson_gxl_ecc_caps,
1363 };
1364
1365 static const struct meson_nfc_data meson_axg_data = {
1366         .ecc_caps = &meson_axg_ecc_caps,
1367 };
1368
1369 static const struct of_device_id meson_nfc_id_table[] = {
1370         {
1371                 .compatible = "amlogic,meson-gxl-nfc",
1372                 .data = &meson_gxl_data,
1373         }, {
1374                 .compatible = "amlogic,meson-axg-nfc",
1375                 .data = &meson_axg_data,
1376         },
1377         {}
1378 };
1379 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1380
1381 static int meson_nfc_probe(struct platform_device *pdev)
1382 {
1383         struct device *dev = &pdev->dev;
1384         struct meson_nfc *nfc;
1385         struct resource *res;
1386         int ret, irq;
1387
1388         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1389         if (!nfc)
1390                 return -ENOMEM;
1391
1392         nfc->data = of_device_get_match_data(&pdev->dev);
1393         if (!nfc->data)
1394                 return -ENODEV;
1395
1396         nand_controller_init(&nfc->controller);
1397         INIT_LIST_HEAD(&nfc->chips);
1398         init_completion(&nfc->completion);
1399
1400         nfc->dev = dev;
1401
1402         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1403         nfc->reg_base = devm_ioremap_resource(dev, res);
1404         if (IS_ERR(nfc->reg_base))
1405                 return PTR_ERR(nfc->reg_base);
1406
1407         nfc->reg_clk =
1408                 syscon_regmap_lookup_by_phandle(dev->of_node,
1409                                                 "amlogic,mmc-syscon");
1410         if (IS_ERR(nfc->reg_clk)) {
1411                 dev_err(dev, "Failed to lookup clock base\n");
1412                 return PTR_ERR(nfc->reg_clk);
1413         }
1414
1415         irq = platform_get_irq(pdev, 0);
1416         if (irq < 0)
1417                 return -EINVAL;
1418
1419         ret = meson_nfc_clk_init(nfc);
1420         if (ret) {
1421                 dev_err(dev, "failed to initialize NAND clock\n");
1422                 return ret;
1423         }
1424
1425         writel(0, nfc->reg_base + NFC_REG_CFG);
1426         ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1427         if (ret) {
1428                 dev_err(dev, "failed to request NFC IRQ\n");
1429                 ret = -EINVAL;
1430                 goto err_clk;
1431         }
1432
1433         ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1434         if (ret) {
1435                 dev_err(dev, "failed to set DMA mask\n");
1436                 goto err_clk;
1437         }
1438
1439         platform_set_drvdata(pdev, nfc);
1440
1441         ret = meson_nfc_nand_chips_init(dev, nfc);
1442         if (ret) {
1443                 dev_err(dev, "failed to init NAND chips\n");
1444                 goto err_clk;
1445         }
1446
1447         return 0;
1448 err_clk:
1449         meson_nfc_disable_clk(nfc);
1450         return ret;
1451 }
1452
1453 static int meson_nfc_remove(struct platform_device *pdev)
1454 {
1455         struct meson_nfc *nfc = platform_get_drvdata(pdev);
1456         int ret;
1457
1458         ret = meson_nfc_nand_chip_cleanup(nfc);
1459         if (ret)
1460                 return ret;
1461
1462         meson_nfc_disable_clk(nfc);
1463
1464         platform_set_drvdata(pdev, NULL);
1465
1466         return 0;
1467 }
1468
1469 static struct platform_driver meson_nfc_driver = {
1470         .probe  = meson_nfc_probe,
1471         .remove = meson_nfc_remove,
1472         .driver = {
1473                 .name  = "meson-nand",
1474                 .of_match_table = meson_nfc_id_table,
1475         },
1476 };
1477 module_platform_driver(meson_nfc_driver);
1478
1479 MODULE_LICENSE("Dual MIT/GPL");
1480 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1481 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");