drivers/mtd/nand/raw/arasan-nand-controller.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Arasan NAND Flash Controller Driver
   4  *
   5  * Copyright (C) 2014 - 2020 Xilinx, Inc.
   6  * Author:
   7  *   Miquel Raynal <miquel.raynal@bootlin.com>
   8  * Original work (fully rewritten):
   9  *   Punnaiah Choudary Kalluri <punnaia@xilinx.com>
  10  *   Naga Sureshkumar Relli <nagasure@xilinx.com>
  11  */
  12
  13 #include <linux/bch.h>
  14 #include <linux/bitfield.h>
  15 #include <linux/clk.h>
  16 #include <linux/delay.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/module.h>
  21 #include <linux/mtd/mtd.h>
  22 #include <linux/mtd/partitions.h>
  23 #include <linux/mtd/rawnand.h>
  24 #include <linux/of.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27
  28 #define PKT_REG                         0x00
  29 #define   PKT_SIZE(x)                   FIELD_PREP(GENMASK(10, 0), (x))
  30 #define   PKT_STEPS(x)                  FIELD_PREP(GENMASK(23, 12), (x))
  31
  32 #define MEM_ADDR1_REG                   0x04
  33
  34 #define MEM_ADDR2_REG                   0x08
  35 #define   ADDR2_STRENGTH(x)             FIELD_PREP(GENMASK(27, 25), (x))
  36 #define   ADDR2_CS(x)                   FIELD_PREP(GENMASK(31, 30), (x))
  37
  38 #define CMD_REG                         0x0C
  39 #define   CMD_1(x)                      FIELD_PREP(GENMASK(7, 0), (x))
  40 #define   CMD_2(x)                      FIELD_PREP(GENMASK(15, 8), (x))
  41 #define   CMD_PAGE_SIZE(x)              FIELD_PREP(GENMASK(25, 23), (x))
  42 #define   CMD_DMA_ENABLE                BIT(27)
  43 #define   CMD_NADDRS(x)                 FIELD_PREP(GENMASK(30, 28), (x))
  44 #define   CMD_ECC_ENABLE                BIT(31)
  45
  46 #define PROG_REG                        0x10
  47 #define   PROG_PGRD                     BIT(0)
  48 #define   PROG_ERASE                    BIT(2)
  49 #define   PROG_STATUS                   BIT(3)
  50 #define   PROG_PGPROG                   BIT(4)
  51 #define   PROG_RDID                     BIT(6)
  52 #define   PROG_RDPARAM                  BIT(7)
  53 #define   PROG_RST                      BIT(8)
  54 #define   PROG_GET_FEATURE              BIT(9)
  55 #define   PROG_SET_FEATURE              BIT(10)
  56
  57 #define INTR_STS_EN_REG                 0x14
  58 #define INTR_SIG_EN_REG                 0x18
  59 #define INTR_STS_REG                    0x1C
  60 #define   WRITE_READY                   BIT(0)
  61 #define   READ_READY                    BIT(1)
  62 #define   XFER_COMPLETE                 BIT(2)
  63 #define   DMA_BOUNDARY                  BIT(6)
  64 #define   EVENT_MASK                    GENMASK(7, 0)
  65
  66 #define READY_STS_REG                   0x20
  67
  68 #define DMA_ADDR0_REG                   0x50
  69 #define DMA_ADDR1_REG                   0x24
  70
  71 #define FLASH_STS_REG                   0x28
  72
  73 #define DATA_PORT_REG                   0x30
  74
  75 #define ECC_CONF_REG                    0x34
  76 #define   ECC_CONF_COL(x)               FIELD_PREP(GENMASK(15, 0), (x))
  77 #define   ECC_CONF_LEN(x)               FIELD_PREP(GENMASK(26, 16), (x))
  78 #define   ECC_CONF_BCH_EN               BIT(27)
  79
  80 #define ECC_ERR_CNT_REG                 0x38
  81 #define   GET_PKT_ERR_CNT(x)            FIELD_GET(GENMASK(7, 0), (x))
  82 #define   GET_PAGE_ERR_CNT(x)           FIELD_GET(GENMASK(16, 8), (x))
  83
  84 #define ECC_SP_REG                      0x3C
  85 #define   ECC_SP_CMD1(x)                FIELD_PREP(GENMASK(7, 0), (x))
  86 #define   ECC_SP_CMD2(x)                FIELD_PREP(GENMASK(15, 8), (x))
  87 #define   ECC_SP_ADDRS(x)               FIELD_PREP(GENMASK(30, 28), (x))
  88
  89 #define ECC_1ERR_CNT_REG                0x40
  90 #define ECC_2ERR_CNT_REG                0x44
  91
  92 #define DATA_INTERFACE_REG              0x6C
  93 #define   DIFACE_SDR_MODE(x)            FIELD_PREP(GENMASK(2, 0), (x))
  94 #define   DIFACE_DDR_MODE(x)            FIELD_PREP(GENMASK(5, 3), (x))
  95 #define   DIFACE_SDR                    0
  96 #define   DIFACE_NVDDR                  BIT(9)
  97
  98 #define ANFC_MAX_CS                     2
  99 #define ANFC_DFLT_TIMEOUT_US            1000000
 100 #define ANFC_MAX_CHUNK_SIZE             SZ_1M
 101 #define ANFC_MAX_PARAM_SIZE             SZ_4K
 102 #define ANFC_MAX_STEPS                  SZ_2K
 103 #define ANFC_MAX_PKT_SIZE               (SZ_2K - 1)
 104 #define ANFC_MAX_ADDR_CYC               5U
 105 #define ANFC_RSVD_ECC_BYTES             21
 106
 107 #define ANFC_XLNX_SDR_DFLT_CORE_CLK     100000000
 108 #define ANFC_XLNX_SDR_HS_CORE_CLK       80000000
 109
 110 /**
 111  * struct anfc_op - Defines how to execute an operation
 112  * @pkt_reg: Packet register
 113  * @addr1_reg: Memory address 1 register
 114  * @addr2_reg: Memory address 2 register
 115  * @cmd_reg: Command register
 116  * @prog_reg: Program register
 117  * @steps: Number of "packets" to read/write
 118  * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
 119  * @len: Data transfer length
 120  * @read: Data transfer direction from the controller point of view
 121  */
 122 struct anfc_op {
 123         u32 pkt_reg;
 124         u32 addr1_reg;
 125         u32 addr2_reg;
 126         u32 cmd_reg;
 127         u32 prog_reg;
 128         int steps;
 129         unsigned int rdy_timeout_ms;
 130         unsigned int len;
 131         bool read;
 132         u8 *buf;
 133 };
 134
 135 /**
 136  * struct anand - Defines the NAND chip related information
 137  * @node:               Used to store NAND chips into a list
 138  * @chip:               NAND chip information structure
 139  * @cs:                 Chip select line
 140  * @rb:                 Ready-busy line
 141  * @page_sz:            Register value of the page_sz field to use
 142  * @clk:                Expected clock frequency to use
 143  * @timings:            Data interface timing mode to use
 144  * @ecc_conf:           Hardware ECC configuration value
 145  * @strength:           Register value of the ECC strength
 146  * @raddr_cycles:       Row address cycle information
 147  * @caddr_cycles:       Column address cycle information
 148  * @ecc_bits:           Exact number of ECC bits per syndrome
 149  * @ecc_total:          Total number of ECC bytes
 150  * @errloc:             Array of errors located with soft BCH
 151  * @hw_ecc:             Buffer to store syndromes computed by hardware
 152  * @bch:                BCH structure
 153  */
 154 struct anand {
 155         struct list_head node;
 156         struct nand_chip chip;
 157         unsigned int cs;
 158         unsigned int rb;
 159         unsigned int page_sz;
 160         unsigned long clk;
 161         u32 timings;
 162         u32 ecc_conf;
 163         u32 strength;
 164         u16 raddr_cycles;
 165         u16 caddr_cycles;
 166         unsigned int ecc_bits;
 167         unsigned int ecc_total;
 168         unsigned int *errloc;
 169         u8 *hw_ecc;
 170         struct bch_control *bch;
 171 };
 172
 173 /**
 174  * struct arasan_nfc - Defines the Arasan NAND flash controller driver instance
 175  * @dev:                Pointer to the device structure
 176  * @base:               Remapped register area
 177  * @controller_clk:             Pointer to the system clock
 178  * @bus_clk:            Pointer to the flash clock
 179  * @controller:         Base controller structure
 180  * @chips:              List of all NAND chips attached to the controller
 181  * @assigned_cs:        Bitmask describing already assigned CS lines
 182  * @cur_clk:            Current clock rate
 183  */
 184 struct arasan_nfc {
 185         struct device *dev;
 186         void __iomem *base;
 187         struct clk *controller_clk;
 188         struct clk *bus_clk;
 189         struct nand_controller controller;
 190         struct list_head chips;
 191         unsigned long assigned_cs;
 192         unsigned int cur_clk;
 193 };
 194
 195 static struct anand *to_anand(struct nand_chip *nand)
 196 {
 197         return container_of(nand, struct anand, chip);
 198 }
 199
 200 static struct arasan_nfc *to_anfc(struct nand_controller *ctrl)
 201 {
 202         return container_of(ctrl, struct arasan_nfc, controller);
 203 }
 204
 205 static int anfc_wait_for_event(struct arasan_nfc *nfc, unsigned int event)
 206 {
 207         u32 val;
 208         int ret;
 209
 210         ret = readl_relaxed_poll_timeout(nfc->base + INTR_STS_REG, val,
 211                                          val & event, 0,
 212                                          ANFC_DFLT_TIMEOUT_US);
 213         if (ret) {
 214                 dev_err(nfc->dev, "Timeout waiting for event 0x%x\n", event);
 215                 return -ETIMEDOUT;
 216         }
 217
 218         writel_relaxed(event, nfc->base + INTR_STS_REG);
 219
 220         return 0;
 221 }
 222
 223 static int anfc_wait_for_rb(struct arasan_nfc *nfc, struct nand_chip *chip,
 224                             unsigned int timeout_ms)
 225 {
 226         struct anand *anand = to_anand(chip);
 227         u32 val;
 228         int ret;
 229
 230         /* There is no R/B interrupt, we must poll a register */
 231         ret = readl_relaxed_poll_timeout(nfc->base + READY_STS_REG, val,
 232                                          val & BIT(anand->rb),
 233                                          1, timeout_ms * 1000);
 234         if (ret) {
 235                 dev_err(nfc->dev, "Timeout waiting for R/B 0x%x\n",
 236                         readl_relaxed(nfc->base + READY_STS_REG));
 237                 return -ETIMEDOUT;
 238         }
 239
 240         return 0;
 241 }
 242
 243 static void anfc_trigger_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
 244 {
 245         writel_relaxed(nfc_op->pkt_reg, nfc->base + PKT_REG);
 246         writel_relaxed(nfc_op->addr1_reg, nfc->base + MEM_ADDR1_REG);
 247         writel_relaxed(nfc_op->addr2_reg, nfc->base + MEM_ADDR2_REG);
 248         writel_relaxed(nfc_op->cmd_reg, nfc->base + CMD_REG);
 249         writel_relaxed(nfc_op->prog_reg, nfc->base + PROG_REG);
 250 }
 251
 252 static int anfc_pkt_len_config(unsigned int len, unsigned int *steps,
 253                                unsigned int *pktsize)
 254 {
 255         unsigned int nb, sz;
 256
 257         for (nb = 1; nb < ANFC_MAX_STEPS; nb *= 2) {
 258                 sz = len / nb;
 259                 if (sz <= ANFC_MAX_PKT_SIZE)
 260                         break;
 261         }
 262
 263         if (sz * nb != len)
 264                 return -ENOTSUPP;
 265
 266         if (steps)
 267                 *steps = nb;
 268
 269         if (pktsize)
 270                 *pktsize = sz;
 271
 272         return 0;
 273 }
 274
 275 static int anfc_select_target(struct nand_chip *chip, int target)
 276 {
 277         struct anand *anand = to_anand(chip);
 278         struct arasan_nfc *nfc = to_anfc(chip->controller);
 279         int ret;
 280
 281         /* Update the controller timings and the potential ECC configuration */
 282         writel_relaxed(anand->timings, nfc->base + DATA_INTERFACE_REG);
 283
 284         /* Update clock frequency */
 285         if (nfc->cur_clk != anand->clk) {
 286                 clk_disable_unprepare(nfc->bus_clk);
 287                 ret = clk_set_rate(nfc->bus_clk, anand->clk);
 288                 if (ret) {
 289                         dev_err(nfc->dev, "Failed to change clock rate\n");
 290                         return ret;
 291                 }
 292
 293                 ret = clk_prepare_enable(nfc->bus_clk);
 294                 if (ret) {
 295                         dev_err(nfc->dev,
 296                                 "Failed to re-enable the bus clock\n");
 297                         return ret;
 298                 }
 299
 300                 nfc->cur_clk = anand->clk;
 301         }
 302
 303         return 0;
 304 }
 305
 306 /*
 307  * When using the embedded hardware ECC engine, the controller is in charge of
 308  * feeding the engine with, first, the ECC residue present in the data array.
 309  * A typical read operation is:
 310  * 1/ Assert the read operation by sending the relevant command/address cycles
 311  *    but targeting the column of the first ECC bytes in the OOB area instead of
 312  *    the main data directly.
 313  * 2/ After having read the relevant number of ECC bytes, the controller uses
 314  *    the RNDOUT/RNDSTART commands which are set into the "ECC Spare Command
 315  *    Register" to move the pointer back at the beginning of the main data.
 316  * 3/ It will read the content of the main area for a given size (pktsize) and
 317  *    will feed the ECC engine with this buffer again.
 318  * 4/ The ECC engine derives the ECC bytes for the given data and compare them
 319  *    with the ones already received. It eventually trigger status flags and
 320  *    then set the "Buffer Read Ready" flag.
 321  * 5/ The corrected data is then available for reading from the data port
 322  *    register.
 323  *
 324  * The hardware BCH ECC engine is known to be inconstent in BCH mode and never
 325  * reports uncorrectable errors. Because of this bug, we have to use the
 326  * software BCH implementation in the read path.
 327  */
 328 static int anfc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
 329                                  int oob_required, int page)
 330 {
 331         struct arasan_nfc *nfc = to_anfc(chip->controller);
 332         struct mtd_info *mtd = nand_to_mtd(chip);
 333         struct anand *anand = to_anand(chip);
 334         unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
 335         unsigned int max_bitflips = 0;
 336         dma_addr_t dma_addr;
 337         int step, ret;
 338         struct anfc_op nfc_op = {
 339                 .pkt_reg =
 340                         PKT_SIZE(chip->ecc.size) |
 341                         PKT_STEPS(chip->ecc.steps),
 342                 .addr1_reg =
 343                         (page & 0xFF) << (8 * (anand->caddr_cycles)) |
 344                         (((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
 345                 .addr2_reg =
 346                         ((page >> 16) & 0xFF) |
 347                         ADDR2_STRENGTH(anand->strength) |
 348                         ADDR2_CS(anand->cs),
 349                 .cmd_reg =
 350                         CMD_1(NAND_CMD_READ0) |
 351                         CMD_2(NAND_CMD_READSTART) |
 352                         CMD_PAGE_SIZE(anand->page_sz) |
 353                         CMD_DMA_ENABLE |
 354                         CMD_NADDRS(anand->caddr_cycles +
 355                                    anand->raddr_cycles),
 356                 .prog_reg = PROG_PGRD,
 357         };
 358
 359         dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_FROM_DEVICE);
 360         if (dma_mapping_error(nfc->dev, dma_addr)) {
 361                 dev_err(nfc->dev, "Buffer mapping error");
 362                 return -EIO;
 363         }
 364
 365         writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
 366         writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
 367
 368         anfc_trigger_op(nfc, &nfc_op);
 369
 370         ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 371         dma_unmap_single(nfc->dev, dma_addr, len, DMA_FROM_DEVICE);
 372         if (ret) {
 373                 dev_err(nfc->dev, "Error reading page %d\n", page);
 374                 return ret;
 375         }
 376
 377         /* Store the raw OOB bytes as well */
 378         ret = nand_change_read_column_op(chip, mtd->writesize, chip->oob_poi,
 379                                          mtd->oobsize, 0);
 380         if (ret)
 381                 return ret;
 382
 383         /*
 384          * For each step, compute by softare the BCH syndrome over the raw data.
 385          * Compare the theoretical amount of errors and compare with the
 386          * hardware engine feedback.
 387          */
 388         for (step = 0; step < chip->ecc.steps; step++) {
 389                 u8 *raw_buf = &buf[step * chip->ecc.size];
 390                 unsigned int bit, byte;
 391                 int bf, i;
 392
 393                 /* Extract the syndrome, it is not necessarily aligned */
 394                 memset(anand->hw_ecc, 0, chip->ecc.bytes);
 395                 nand_extract_bits(anand->hw_ecc, 0,
 396                                   &chip->oob_poi[mtd->oobsize - anand->ecc_total],
 397                                   anand->ecc_bits * step, anand->ecc_bits);
 398
 399                 bf = bch_decode(anand->bch, raw_buf, chip->ecc.size,
 400                                 anand->hw_ecc, NULL, NULL, anand->errloc);
 401                 if (!bf) {
 402                         continue;
 403                 } else if (bf > 0) {
 404                         for (i = 0; i < bf; i++) {
 405                                 /* Only correct the data, not the syndrome */
 406                                 if (anand->errloc[i] < (chip->ecc.size * 8)) {
 407                                         bit = BIT(anand->errloc[i] & 7);
 408                                         byte = anand->errloc[i] >> 3;
 409                                         raw_buf[byte] ^= bit;
 410                                 }
 411                         }
 412
 413                         mtd->ecc_stats.corrected += bf;
 414                         max_bitflips = max_t(unsigned int, max_bitflips, bf);
 415
 416                         continue;
 417                 }
 418
 419                 bf = nand_check_erased_ecc_chunk(raw_buf, chip->ecc.size,
 420                                                  NULL, 0, NULL, 0,
 421                                                  chip->ecc.strength);
 422                 if (bf > 0) {
 423                         mtd->ecc_stats.corrected += bf;
 424                         max_bitflips = max_t(unsigned int, max_bitflips, bf);
 425                         memset(raw_buf, 0xFF, chip->ecc.size);
 426                 } else if (bf < 0) {
 427                         mtd->ecc_stats.failed++;
 428                 }
 429         }
 430
 431         return 0;
 432 }
 433
 434 static int anfc_sel_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
 435                                      int oob_required, int page)
 436 {
 437         int ret;
 438
 439         ret = anfc_select_target(chip, chip->cur_cs);
 440         if (ret)
 441                 return ret;
 442
 443         return anfc_read_page_hw_ecc(chip, buf, oob_required, page);
 444 };
 445
 446 static int anfc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
 447                                   int oob_required, int page)
 448 {
 449         struct anand *anand = to_anand(chip);
 450         struct arasan_nfc *nfc = to_anfc(chip->controller);
 451         struct mtd_info *mtd = nand_to_mtd(chip);
 452         unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
 453         dma_addr_t dma_addr;
 454         u8 status;
 455         int ret;
 456         struct anfc_op nfc_op = {
 457                 .pkt_reg =
 458                         PKT_SIZE(chip->ecc.size) |
 459                         PKT_STEPS(chip->ecc.steps),
 460                 .addr1_reg =
 461                         (page & 0xFF) << (8 * (anand->caddr_cycles)) |
 462                         (((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
 463                 .addr2_reg =
 464                         ((page >> 16) & 0xFF) |
 465                         ADDR2_STRENGTH(anand->strength) |
 466                         ADDR2_CS(anand->cs),
 467                 .cmd_reg =
 468                         CMD_1(NAND_CMD_SEQIN) |
 469                         CMD_2(NAND_CMD_PAGEPROG) |
 470                         CMD_PAGE_SIZE(anand->page_sz) |
 471                         CMD_DMA_ENABLE |
 472                         CMD_NADDRS(anand->caddr_cycles +
 473                                    anand->raddr_cycles) |
 474                         CMD_ECC_ENABLE,
 475                 .prog_reg = PROG_PGPROG,
 476         };
 477
 478         writel_relaxed(anand->ecc_conf, nfc->base + ECC_CONF_REG);
 479         writel_relaxed(ECC_SP_CMD1(NAND_CMD_RNDIN) |
 480                        ECC_SP_ADDRS(anand->caddr_cycles),
 481                        nfc->base + ECC_SP_REG);
 482
 483         dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_TO_DEVICE);
 484         if (dma_mapping_error(nfc->dev, dma_addr)) {
 485                 dev_err(nfc->dev, "Buffer mapping error");
 486                 return -EIO;
 487         }
 488
 489         writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
 490         writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
 491
 492         anfc_trigger_op(nfc, &nfc_op);
 493         ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 494         dma_unmap_single(nfc->dev, dma_addr, len, DMA_TO_DEVICE);
 495         if (ret) {
 496                 dev_err(nfc->dev, "Error writing page %d\n", page);
 497                 return ret;
 498         }
 499
 500         /* Spare data is not protected */
 501         if (oob_required) {
 502                 ret = nand_write_oob_std(chip, page);
 503                 if (ret)
 504                         return ret;
 505         }
 506
 507         /* Check write status on the chip side */
 508         ret = nand_status_op(chip, &status);
 509         if (ret)
 510                 return ret;
 511
 512         if (status & NAND_STATUS_FAIL)
 513                 return -EIO;
 514
 515         return 0;
 516 }
 517
 518 static int anfc_sel_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
 519                                       int oob_required, int page)
 520 {
 521         int ret;
 522
 523         ret = anfc_select_target(chip, chip->cur_cs);
 524         if (ret)
 525                 return ret;
 526
 527         return anfc_write_page_hw_ecc(chip, buf, oob_required, page);
 528 };
 529
 530 /* NAND framework ->exec_op() hooks and related helpers */
 531 static int anfc_parse_instructions(struct nand_chip *chip,
 532                                    const struct nand_subop *subop,
 533                                    struct anfc_op *nfc_op)
 534 {
 535         struct anand *anand = to_anand(chip);
 536         const struct nand_op_instr *instr = NULL;
 537         bool first_cmd = true;
 538         unsigned int op_id;
 539         int ret, i;
 540
 541         memset(nfc_op, 0, sizeof(*nfc_op));
 542         nfc_op->addr2_reg = ADDR2_CS(anand->cs);
 543         nfc_op->cmd_reg = CMD_PAGE_SIZE(anand->page_sz);
 544
 545         for (op_id = 0; op_id < subop->ninstrs; op_id++) {
 546                 unsigned int offset, naddrs, pktsize;
 547                 const u8 *addrs;
 548                 u8 *buf;
 549
 550                 instr = &subop->instrs[op_id];
 551
 552                 switch (instr->type) {
 553                 case NAND_OP_CMD_INSTR:
 554                         if (first_cmd)
 555                                 nfc_op->cmd_reg |= CMD_1(instr->ctx.cmd.opcode);
 556                         else
 557                                 nfc_op->cmd_reg |= CMD_2(instr->ctx.cmd.opcode);
 558
 559                         first_cmd = false;
 560                         break;
 561
 562                 case NAND_OP_ADDR_INSTR:
 563                         offset = nand_subop_get_addr_start_off(subop, op_id);
 564                         naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
 565                         addrs = &instr->ctx.addr.addrs[offset];
 566                         nfc_op->cmd_reg |= CMD_NADDRS(naddrs);
 567
 568                         for (i = 0; i < min(ANFC_MAX_ADDR_CYC, naddrs); i++) {
 569                                 if (i < 4)
 570                                         nfc_op->addr1_reg |= (u32)addrs[i] << i * 8;
 571                                 else
 572                                         nfc_op->addr2_reg |= addrs[i];
 573                         }
 574
 575                         break;
 576                 case NAND_OP_DATA_IN_INSTR:
 577                         nfc_op->read = true;
 578                         fallthrough;
 579                 case NAND_OP_DATA_OUT_INSTR:
 580                         offset = nand_subop_get_data_start_off(subop, op_id);
 581                         buf = instr->ctx.data.buf.in;
 582                         nfc_op->buf = &buf[offset];
 583                         nfc_op->len = nand_subop_get_data_len(subop, op_id);
 584                         ret = anfc_pkt_len_config(nfc_op->len, &nfc_op->steps,
 585                                                   &pktsize);
 586                         if (ret)
 587                                 return ret;
 588
 589                         /*
 590                          * Number of DATA cycles must be aligned on 4, this
 591                          * means the controller might read/write more than
 592                          * requested. This is harmless most of the time as extra
 593                          * DATA are discarded in the write path and read pointer
 594                          * adjusted in the read path.
 595                          *
 596                          * FIXME: The core should mark operations where
 597                          * reading/writing more is allowed so the exec_op()
 598                          * implementation can take the right decision when the
 599                          * alignment constraint is not met: adjust the number of
 600                          * DATA cycles when it's allowed, reject the operation
 601                          * otherwise.
 602                          */
 603                         nfc_op->pkt_reg |= PKT_SIZE(round_up(pktsize, 4)) |
 604                                            PKT_STEPS(nfc_op->steps);
 605                         break;
 606                 case NAND_OP_WAITRDY_INSTR:
 607                         nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
 608                         break;
 609                 }
 610         }
 611
 612         return 0;
 613 }
 614
 615 static int anfc_rw_pio_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
 616 {
 617         unsigned int dwords = (nfc_op->len / 4) / nfc_op->steps;
 618         unsigned int last_len = nfc_op->len % 4;
 619         unsigned int offset, dir;
 620         u8 *buf = nfc_op->buf;
 621         int ret, i;
 622
 623         for (i = 0; i < nfc_op->steps; i++) {
 624                 dir = nfc_op->read ? READ_READY : WRITE_READY;
 625                 ret = anfc_wait_for_event(nfc, dir);
 626                 if (ret) {
 627                         dev_err(nfc->dev, "PIO %s ready signal not received\n",
 628                                 nfc_op->read ? "Read" : "Write");
 629                         return ret;
 630                 }
 631
 632                 offset = i * (dwords * 4);
 633                 if (nfc_op->read)
 634                         ioread32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
 635                                      dwords);
 636                 else
 637                         iowrite32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
 638                                       dwords);
 639         }
 640
 641         if (last_len) {
 642                 u32 remainder;
 643
 644                 offset = nfc_op->len - last_len;
 645
 646                 if (nfc_op->read) {
 647                         remainder = readl_relaxed(nfc->base + DATA_PORT_REG);
 648                         memcpy(&buf[offset], &remainder, last_len);
 649                 } else {
 650                         memcpy(&remainder, &buf[offset], last_len);
 651                         writel_relaxed(remainder, nfc->base + DATA_PORT_REG);
 652                 }
 653         }
 654
 655         return anfc_wait_for_event(nfc, XFER_COMPLETE);
 656 }
 657
 658 static int anfc_misc_data_type_exec(struct nand_chip *chip,
 659                                     const struct nand_subop *subop,
 660                                     u32 prog_reg)
 661 {
 662         struct arasan_nfc *nfc = to_anfc(chip->controller);
 663         struct anfc_op nfc_op = {};
 664         int ret;
 665
 666         ret = anfc_parse_instructions(chip, subop, &nfc_op);
 667         if (ret)
 668                 return ret;
 669
 670         nfc_op.prog_reg = prog_reg;
 671         anfc_trigger_op(nfc, &nfc_op);
 672
 673         if (nfc_op.rdy_timeout_ms) {
 674                 ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 675                 if (ret)
 676                         return ret;
 677         }
 678
 679         return anfc_rw_pio_op(nfc, &nfc_op);
 680 }
 681
 682 static int anfc_param_read_type_exec(struct nand_chip *chip,
 683                                      const struct nand_subop *subop)
 684 {
 685         return anfc_misc_data_type_exec(chip, subop, PROG_RDPARAM);
 686 }
 687
 688 static int anfc_data_read_type_exec(struct nand_chip *chip,
 689                                     const struct nand_subop *subop)
 690 {
 691         return anfc_misc_data_type_exec(chip, subop, PROG_PGRD);
 692 }
 693
 694 static int anfc_param_write_type_exec(struct nand_chip *chip,
 695                                       const struct nand_subop *subop)
 696 {
 697         return anfc_misc_data_type_exec(chip, subop, PROG_SET_FEATURE);
 698 }
 699
 700 static int anfc_data_write_type_exec(struct nand_chip *chip,
 701                                      const struct nand_subop *subop)
 702 {
 703         return anfc_misc_data_type_exec(chip, subop, PROG_PGPROG);
 704 }
 705
 706 static int anfc_misc_zerolen_type_exec(struct nand_chip *chip,
 707                                        const struct nand_subop *subop,
 708                                        u32 prog_reg)
 709 {
 710         struct arasan_nfc *nfc = to_anfc(chip->controller);
 711         struct anfc_op nfc_op = {};
 712         int ret;
 713
 714         ret = anfc_parse_instructions(chip, subop, &nfc_op);
 715         if (ret)
 716                 return ret;
 717
 718         nfc_op.prog_reg = prog_reg;
 719         anfc_trigger_op(nfc, &nfc_op);
 720
 721         ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 722         if (ret)
 723                 return ret;
 724
 725         if (nfc_op.rdy_timeout_ms)
 726                 ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 727
 728         return ret;
 729 }
 730
 731 static int anfc_status_type_exec(struct nand_chip *chip,
 732                                  const struct nand_subop *subop)
 733 {
 734         struct arasan_nfc *nfc = to_anfc(chip->controller);
 735         u32 tmp;
 736         int ret;
 737
 738         /* See anfc_check_op() for details about this constraint */
 739         if (subop->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS)
 740                 return -ENOTSUPP;
 741
 742         ret = anfc_misc_zerolen_type_exec(chip, subop, PROG_STATUS);
 743         if (ret)
 744                 return ret;
 745
 746         tmp = readl_relaxed(nfc->base + FLASH_STS_REG);
 747         memcpy(subop->instrs[1].ctx.data.buf.in, &tmp, 1);
 748
 749         return 0;
 750 }
 751
 752 static int anfc_reset_type_exec(struct nand_chip *chip,
 753                                 const struct nand_subop *subop)
 754 {
 755         return anfc_misc_zerolen_type_exec(chip, subop, PROG_RST);
 756 }
 757
 758 static int anfc_erase_type_exec(struct nand_chip *chip,
 759                                 const struct nand_subop *subop)
 760 {
 761         return anfc_misc_zerolen_type_exec(chip, subop, PROG_ERASE);
 762 }
 763
 764 static int anfc_wait_type_exec(struct nand_chip *chip,
 765                                const struct nand_subop *subop)
 766 {
 767         struct arasan_nfc *nfc = to_anfc(chip->controller);
 768         struct anfc_op nfc_op = {};
 769         int ret;
 770
 771         ret = anfc_parse_instructions(chip, subop, &nfc_op);
 772         if (ret)
 773                 return ret;
 774
 775         return anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 776 }
 777
 778 static const struct nand_op_parser anfc_op_parser = NAND_OP_PARSER(
 779         NAND_OP_PARSER_PATTERN(
 780                 anfc_param_read_type_exec,
 781                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 782                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 783                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 784                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
 785         NAND_OP_PARSER_PATTERN(
 786                 anfc_param_write_type_exec,
 787                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 788                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 789                 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_PARAM_SIZE)),
 790         NAND_OP_PARSER_PATTERN(
 791                 anfc_data_read_type_exec,
 792                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 793                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 794                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 795                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 796                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, ANFC_MAX_CHUNK_SIZE)),
 797         NAND_OP_PARSER_PATTERN(
 798                 anfc_data_write_type_exec,
 799                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 800                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 801                 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_CHUNK_SIZE),
 802                 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
 803         NAND_OP_PARSER_PATTERN(
 804                 anfc_reset_type_exec,
 805                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 806                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 807         NAND_OP_PARSER_PATTERN(
 808                 anfc_erase_type_exec,
 809                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 810                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 811                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 812                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 813         NAND_OP_PARSER_PATTERN(
 814                 anfc_status_type_exec,
 815                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
 816                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
 817         NAND_OP_PARSER_PATTERN(
 818                 anfc_wait_type_exec,
 819                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 820         );
 821
 822 static int anfc_check_op(struct nand_chip *chip,
 823                          const struct nand_operation *op)
 824 {
 825         const struct nand_op_instr *instr;
 826         int op_id;
 827
 828         /*
 829          * The controller abstracts all the NAND operations and do not support
 830          * data only operations.
 831          *
 832          * TODO: The nand_op_parser framework should be extended to
 833          * support custom checks on DATA instructions.
 834          */
 835         for (op_id = 0; op_id < op->ninstrs; op_id++) {
 836                 instr = &op->instrs[op_id];
 837
 838                 switch (instr->type) {
 839                 case NAND_OP_ADDR_INSTR:
 840                         if (instr->ctx.addr.naddrs > ANFC_MAX_ADDR_CYC)
 841                                 return -ENOTSUPP;
 842
 843                         break;
 844                 case NAND_OP_DATA_IN_INSTR:
 845                 case NAND_OP_DATA_OUT_INSTR:
 846                         if (instr->ctx.data.len > ANFC_MAX_CHUNK_SIZE)
 847                                 return -ENOTSUPP;
 848
 849                         if (anfc_pkt_len_config(instr->ctx.data.len, 0, 0))
 850                                 return -ENOTSUPP;
 851
 852                         break;
 853                 default:
 854                         break;
 855                 }
 856         }
 857
 858         /*
 859          * The controller does not allow to proceed with a CMD+DATA_IN cycle
 860          * manually on the bus by reading data from the data register. Instead,
 861          * the controller abstract a status read operation with its own status
 862          * register after ordering a read status operation. Hence, we cannot
 863          * support any CMD+DATA_IN operation other than a READ STATUS.
 864          *
 865          * TODO: The nand_op_parser() framework should be extended to describe
 866          * fixed patterns instead of open-coding this check here.
 867          */
 868         if (op->ninstrs == 2 &&
 869             op->instrs[0].type == NAND_OP_CMD_INSTR &&
 870             op->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS &&
 871             op->instrs[1].type == NAND_OP_DATA_IN_INSTR)
 872                 return -ENOTSUPP;
 873
 874         return nand_op_parser_exec_op(chip, &anfc_op_parser, op, true);
 875 }
 876
 877 static int anfc_exec_op(struct nand_chip *chip,
 878                         const struct nand_operation *op,
 879                         bool check_only)
 880 {
 881         int ret;
 882
 883         if (check_only)
 884                 return anfc_check_op(chip, op);
 885
 886         ret = anfc_select_target(chip, op->cs);
 887         if (ret)
 888                 return ret;
 889
 890         return nand_op_parser_exec_op(chip, &anfc_op_parser, op, check_only);
 891 }
 892
 893 static int anfc_setup_interface(struct nand_chip *chip, int target,
 894                                 const struct nand_interface_config *conf)
 895 {
 896         struct anand *anand = to_anand(chip);
 897         struct arasan_nfc *nfc = to_anfc(chip->controller);
 898         struct device_node *np = nfc->dev->of_node;
 899         const struct nand_sdr_timings *sdr;
 900         const struct nand_nvddr_timings *nvddr;
 901
 902         if (nand_interface_is_nvddr(conf)) {
 903                 nvddr = nand_get_nvddr_timings(conf);
 904                 if (IS_ERR(nvddr))
 905                         return PTR_ERR(nvddr);
 906
 907                 /*
 908                  * The controller only supports data payload requests which are
 909                  * a multiple of 4. In practice, most data accesses are 4-byte
 910                  * aligned and this is not an issue. However, rounding up will
 911                  * simply be refused by the controller if we reached the end of
 912                  * the device *and* we are using the NV-DDR interface(!). In
 913                  * this situation, unaligned data requests ending at the device
 914                  * boundary will confuse the controller and cannot be performed.
 915                  *
 916                  * This is something that happens in nand_read_subpage() when
 917                  * selecting software ECC support and must be avoided.
 918                  */
 919                 if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT)
 920                         return -ENOTSUPP;
 921         } else {
 922                 sdr = nand_get_sdr_timings(conf);
 923                 if (IS_ERR(sdr))
 924                         return PTR_ERR(sdr);
 925         }
 926
 927         if (target < 0)
 928                 return 0;
 929
 930         if (nand_interface_is_sdr(conf))
 931                 anand->timings = DIFACE_SDR |
 932                                  DIFACE_SDR_MODE(conf->timings.mode);
 933         else
 934                 anand->timings = DIFACE_NVDDR |
 935                                  DIFACE_DDR_MODE(conf->timings.mode);
 936
 937         if (nand_interface_is_sdr(conf)) {
 938                 anand->clk = ANFC_XLNX_SDR_DFLT_CORE_CLK;
 939         } else {
 940                 /* ONFI timings are defined in picoseconds */
 941                 anand->clk = div_u64((u64)NSEC_PER_SEC * 1000,
 942                                      conf->timings.nvddr.tCK_min);
 943         }
 944
 945         /*
 946          * Due to a hardware bug in the ZynqMP SoC, SDR timing modes 0-1 work
 947          * with f > 90MHz (default clock is 100MHz) but signals are unstable
 948          * with higher modes. Hence we decrease a little bit the clock rate to
 949          * 80MHz when using SDR modes 2-5 with this SoC.
 950          */
 951         if (of_device_is_compatible(np, "xlnx,zynqmp-nand-controller") &&
 952             nand_interface_is_sdr(conf) && conf->timings.mode >= 2)
 953                 anand->clk = ANFC_XLNX_SDR_HS_CORE_CLK;
 954
 955         return 0;
 956 }
 957
 958 static int anfc_calc_hw_ecc_bytes(int step_size, int strength)
 959 {
 960         unsigned int bch_gf_mag, ecc_bits;
 961
 962         switch (step_size) {
 963         case SZ_512:
 964                 bch_gf_mag = 13;
 965                 break;
 966         case SZ_1K:
 967                 bch_gf_mag = 14;
 968                 break;
 969         default:
 970                 return -EINVAL;
 971         }
 972
 973         ecc_bits = bch_gf_mag * strength;
 974
 975         return DIV_ROUND_UP(ecc_bits, 8);
 976 }
 977
 978 static const int anfc_hw_ecc_512_strengths[] = {4, 8, 12};
 979
 980 static const int anfc_hw_ecc_1024_strengths[] = {24};
 981
 982 static const struct nand_ecc_step_info anfc_hw_ecc_step_infos[] = {
 983         {
 984                 .stepsize = SZ_512,
 985                 .strengths = anfc_hw_ecc_512_strengths,
 986                 .nstrengths = ARRAY_SIZE(anfc_hw_ecc_512_strengths),
 987         },
 988         {
 989                 .stepsize = SZ_1K,
 990                 .strengths = anfc_hw_ecc_1024_strengths,
 991                 .nstrengths = ARRAY_SIZE(anfc_hw_ecc_1024_strengths),
 992         },
 993 };
 994
 995 static const struct nand_ecc_caps anfc_hw_ecc_caps = {
 996         .stepinfos = anfc_hw_ecc_step_infos,
 997         .nstepinfos = ARRAY_SIZE(anfc_hw_ecc_step_infos),
 998         .calc_ecc_bytes = anfc_calc_hw_ecc_bytes,
 999 };
1000
1001 static int anfc_init_hw_ecc_controller(struct arasan_nfc *nfc,
1002                                        struct nand_chip *chip)
1003 {
1004         struct anand *anand = to_anand(chip);
1005         struct mtd_info *mtd = nand_to_mtd(chip);
1006         struct nand_ecc_ctrl *ecc = &chip->ecc;
1007         unsigned int bch_prim_poly = 0, bch_gf_mag = 0, ecc_offset;
1008         int ret;
1009
1010         switch (mtd->writesize) {
1011         case SZ_512:
1012         case SZ_2K:
1013         case SZ_4K:
1014         case SZ_8K:
1015         case SZ_16K:
1016                 break;
1017         default:
1018                 dev_err(nfc->dev, "Unsupported page size %d\n", mtd->writesize);
1019                 return -EINVAL;
1020         }
1021
1022         ret = nand_ecc_choose_conf(chip, &anfc_hw_ecc_caps, mtd->oobsize);
1023         if (ret)
1024                 return ret;
1025
1026         switch (ecc->strength) {
1027         case 12:
1028                 anand->strength = 0x1;
1029                 break;
1030         case 8:
1031                 anand->strength = 0x2;
1032                 break;
1033         case 4:
1034                 anand->strength = 0x3;
1035                 break;
1036         case 24:
1037                 anand->strength = 0x4;
1038                 break;
1039         default:
1040                 dev_err(nfc->dev, "Unsupported strength %d\n", ecc->strength);
1041                 return -EINVAL;
1042         }
1043
1044         switch (ecc->size) {
1045         case SZ_512:
1046                 bch_gf_mag = 13;
1047                 bch_prim_poly = 0x201b;
1048                 break;
1049         case SZ_1K:
1050                 bch_gf_mag = 14;
1051                 bch_prim_poly = 0x4443;
1052                 break;
1053         default:
1054                 dev_err(nfc->dev, "Unsupported step size %d\n", ecc->strength);
1055                 return -EINVAL;
1056         }
1057
1058         mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
1059
1060         ecc->steps = mtd->writesize / ecc->size;
1061         ecc->algo = NAND_ECC_ALGO_BCH;
1062         anand->ecc_bits = bch_gf_mag * ecc->strength;
1063         ecc->bytes = DIV_ROUND_UP(anand->ecc_bits, 8);
1064         anand->ecc_total = DIV_ROUND_UP(anand->ecc_bits * ecc->steps, 8);
1065         ecc_offset = mtd->writesize + mtd->oobsize - anand->ecc_total;
1066         anand->ecc_conf = ECC_CONF_COL(ecc_offset) |
1067                           ECC_CONF_LEN(anand->ecc_total) |
1068                           ECC_CONF_BCH_EN;
1069
1070         anand->errloc = devm_kmalloc_array(nfc->dev, ecc->strength,
1071                                            sizeof(*anand->errloc), GFP_KERNEL);
1072         if (!anand->errloc)
1073                 return -ENOMEM;
1074
1075         anand->hw_ecc = devm_kmalloc(nfc->dev, ecc->bytes, GFP_KERNEL);
1076         if (!anand->hw_ecc)
1077                 return -ENOMEM;
1078
1079         /* Enforce bit swapping to fit the hardware */
1080         anand->bch = bch_init(bch_gf_mag, ecc->strength, bch_prim_poly, true);
1081         if (!anand->bch)
1082                 return -EINVAL;
1083
1084         ecc->read_page = anfc_sel_read_page_hw_ecc;
1085         ecc->write_page = anfc_sel_write_page_hw_ecc;
1086
1087         return 0;
1088 }
1089
1090 static int anfc_attach_chip(struct nand_chip *chip)
1091 {
1092         struct anand *anand = to_anand(chip);
1093         struct arasan_nfc *nfc = to_anfc(chip->controller);
1094         struct mtd_info *mtd = nand_to_mtd(chip);
1095         int ret = 0;
1096
1097         if (mtd->writesize <= SZ_512)
1098                 anand->caddr_cycles = 1;
1099         else
1100                 anand->caddr_cycles = 2;
1101
1102         if (chip->options & NAND_ROW_ADDR_3)
1103                 anand->raddr_cycles = 3;
1104         else
1105                 anand->raddr_cycles = 2;
1106
1107         switch (mtd->writesize) {
1108         case 512:
1109                 anand->page_sz = 0;
1110                 break;
1111         case 1024:
1112                 anand->page_sz = 5;
1113                 break;
1114         case 2048:
1115                 anand->page_sz = 1;
1116                 break;
1117         case 4096:
1118                 anand->page_sz = 2;
1119                 break;
1120         case 8192:
1121                 anand->page_sz = 3;
1122                 break;
1123         case 16384:
1124                 anand->page_sz = 4;
1125                 break;
1126         default:
1127                 return -EINVAL;
1128         }
1129
1130         /* These hooks are valid for all ECC providers */
1131         chip->ecc.read_page_raw = nand_monolithic_read_page_raw;
1132         chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
1133
1134         switch (chip->ecc.engine_type) {
1135         case NAND_ECC_ENGINE_TYPE_NONE:
1136         case NAND_ECC_ENGINE_TYPE_SOFT:
1137         case NAND_ECC_ENGINE_TYPE_ON_DIE:
1138                 break;
1139         case NAND_ECC_ENGINE_TYPE_ON_HOST:
1140                 ret = anfc_init_hw_ecc_controller(nfc, chip);
1141                 break;
1142         default:
1143                 dev_err(nfc->dev, "Unsupported ECC mode: %d\n",
1144                         chip->ecc.engine_type);
1145                 return -EINVAL;
1146         }
1147
1148         return ret;
1149 }
1150
1151 static void anfc_detach_chip(struct nand_chip *chip)
1152 {
1153         struct anand *anand = to_anand(chip);
1154
1155         if (anand->bch)
1156                 bch_free(anand->bch);
1157 }
1158
1159 static const struct nand_controller_ops anfc_ops = {
1160         .exec_op = anfc_exec_op,
1161         .setup_interface = anfc_setup_interface,
1162         .attach_chip = anfc_attach_chip,
1163         .detach_chip = anfc_detach_chip,
1164 };
1165
1166 static int anfc_chip_init(struct arasan_nfc *nfc, struct device_node *np)
1167 {
1168         struct anand *anand;
1169         struct nand_chip *chip;
1170         struct mtd_info *mtd;
1171         int cs, rb, ret;
1172
1173         anand = devm_kzalloc(nfc->dev, sizeof(*anand), GFP_KERNEL);
1174         if (!anand)
1175                 return -ENOMEM;
1176
1177         /* We do not support multiple CS per chip yet */
1178         if (of_property_count_elems_of_size(np, "reg", sizeof(u32)) != 1) {
1179                 dev_err(nfc->dev, "Invalid reg property\n");
1180                 return -EINVAL;
1181         }
1182
1183         ret = of_property_read_u32(np, "reg", &cs);
1184         if (ret)
1185                 return ret;
1186
1187         ret = of_property_read_u32(np, "nand-rb", &rb);
1188         if (ret)
1189                 return ret;
1190
1191         if (cs >= ANFC_MAX_CS || rb >= ANFC_MAX_CS) {
1192                 dev_err(nfc->dev, "Wrong CS %d or RB %d\n", cs, rb);
1193                 return -EINVAL;
1194         }
1195
1196         if (test_and_set_bit(cs, &nfc->assigned_cs)) {
1197                 dev_err(nfc->dev, "Already assigned CS %d\n", cs);
1198                 return -EINVAL;
1199         }
1200
1201         anand->cs = cs;
1202         anand->rb = rb;
1203
1204         chip = &anand->chip;
1205         mtd = nand_to_mtd(chip);
1206         mtd->dev.parent = nfc->dev;
1207         chip->controller = &nfc->controller;
1208         chip->options = NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE |
1209                         NAND_USES_DMA;
1210
1211         nand_set_flash_node(chip, np);
1212         if (!mtd->name) {
1213                 dev_err(nfc->dev, "NAND label property is mandatory\n");
1214                 return -EINVAL;
1215         }
1216
1217         ret = nand_scan(chip, 1);
1218         if (ret) {
1219                 dev_err(nfc->dev, "Scan operation failed\n");
1220                 return ret;
1221         }
1222
1223         ret = mtd_device_register(mtd, NULL, 0);
1224         if (ret) {
1225                 nand_cleanup(chip);
1226                 return ret;
1227         }
1228
1229         list_add_tail(&anand->node, &nfc->chips);
1230
1231         return 0;
1232 }
1233
1234 static void anfc_chips_cleanup(struct arasan_nfc *nfc)
1235 {
1236         struct anand *anand, *tmp;
1237         struct nand_chip *chip;
1238         int ret;
1239
1240         list_for_each_entry_safe(anand, tmp, &nfc->chips, node) {
1241                 chip = &anand->chip;
1242                 ret = mtd_device_unregister(nand_to_mtd(chip));
1243                 WARN_ON(ret);
1244                 nand_cleanup(chip);
1245                 list_del(&anand->node);
1246         }
1247 }
1248
1249 static int anfc_chips_init(struct arasan_nfc *nfc)
1250 {
1251         struct device_node *np = nfc->dev->of_node, *nand_np;
1252         int nchips = of_get_child_count(np);
1253         int ret;
1254
1255         if (!nchips || nchips > ANFC_MAX_CS) {
1256                 dev_err(nfc->dev, "Incorrect number of NAND chips (%d)\n",
1257                         nchips);
1258                 return -EINVAL;
1259         }
1260
1261         for_each_child_of_node(np, nand_np) {
1262                 ret = anfc_chip_init(nfc, nand_np);
1263                 if (ret) {
1264                         of_node_put(nand_np);
1265                         anfc_chips_cleanup(nfc);
1266                         break;
1267                 }
1268         }
1269
1270         return ret;
1271 }
1272
1273 static void anfc_reset(struct arasan_nfc *nfc)
1274 {
1275         /* Disable interrupt signals */
1276         writel_relaxed(0, nfc->base + INTR_SIG_EN_REG);
1277
1278         /* Enable interrupt status */
1279         writel_relaxed(EVENT_MASK, nfc->base + INTR_STS_EN_REG);
1280 }
1281
1282 static int anfc_probe(struct platform_device *pdev)
1283 {
1284         struct arasan_nfc *nfc;
1285         int ret;
1286
1287         nfc = devm_kzalloc(&pdev->dev, sizeof(*nfc), GFP_KERNEL);
1288         if (!nfc)
1289                 return -ENOMEM;
1290
1291         nfc->dev = &pdev->dev;
1292         nand_controller_init(&nfc->controller);
1293         nfc->controller.ops = &anfc_ops;
1294         INIT_LIST_HEAD(&nfc->chips);
1295
1296         nfc->base = devm_platform_ioremap_resource(pdev, 0);
1297         if (IS_ERR(nfc->base))
1298                 return PTR_ERR(nfc->base);
1299
1300         anfc_reset(nfc);
1301
1302         nfc->controller_clk = devm_clk_get(&pdev->dev, "controller");
1303         if (IS_ERR(nfc->controller_clk))
1304                 return PTR_ERR(nfc->controller_clk);
1305
1306         nfc->bus_clk = devm_clk_get(&pdev->dev, "bus");
1307         if (IS_ERR(nfc->bus_clk))
1308                 return PTR_ERR(nfc->bus_clk);
1309
1310         ret = clk_prepare_enable(nfc->controller_clk);
1311         if (ret)
1312                 return ret;
1313
1314         ret = clk_prepare_enable(nfc->bus_clk);
1315         if (ret)
1316                 goto disable_controller_clk;
1317
1318         ret = anfc_chips_init(nfc);
1319         if (ret)
1320                 goto disable_bus_clk;
1321
1322         platform_set_drvdata(pdev, nfc);
1323
1324         return 0;
1325
1326 disable_bus_clk:
1327         clk_disable_unprepare(nfc->bus_clk);
1328
1329 disable_controller_clk:
1330         clk_disable_unprepare(nfc->controller_clk);
1331
1332         return ret;
1333 }
1334
1335 static int anfc_remove(struct platform_device *pdev)
1336 {
1337         struct arasan_nfc *nfc = platform_get_drvdata(pdev);
1338
1339         anfc_chips_cleanup(nfc);
1340
1341         clk_disable_unprepare(nfc->bus_clk);
1342         clk_disable_unprepare(nfc->controller_clk);
1343
1344         return 0;
1345 }
1346
1347 static const struct of_device_id anfc_ids[] = {
1348         {
1349                 .compatible = "xlnx,zynqmp-nand-controller",
1350         },
1351         {
1352                 .compatible = "arasan,nfc-v3p10",
1353         },
1354         {}
1355 };
1356 MODULE_DEVICE_TABLE(of, anfc_ids);
1357
1358 static struct platform_driver anfc_driver = {
1359         .driver = {
1360                 .name = "arasan-nand-controller",
1361                 .of_match_table = anfc_ids,
1362         },
1363         .probe = anfc_probe,
1364         .remove = anfc_remove,
1365 };
1366 module_platform_driver(anfc_driver);
1367
1368 MODULE_LICENSE("GPL v2");
1369 MODULE_AUTHOR("Punnaiah Choudary Kalluri <punnaia@xilinx.com>");
1370 MODULE_AUTHOR("Naga Sureshkumar Relli <nagasure@xilinx.com>");
1371 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1372 MODULE_DESCRIPTION("Arasan NAND Flash Controller Driver");