drivers/mtd/nand/raw/qcom_nandc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   4  */
   5 #include <linux/clk.h>
   6 #include <linux/slab.h>
   7 #include <linux/bitops.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/dmaengine.h>
  10 #include <linux/module.h>
  11 #include <linux/mtd/rawnand.h>
  12 #include <linux/mtd/partitions.h>
  13 #include <linux/of.h>
  14 #include <linux/of_device.h>
  15 #include <linux/delay.h>
  16 #include <linux/dma/qcom_bam_dma.h>
  17
  18 /* NANDc reg offsets */
  19 #define NAND_FLASH_CMD                  0x00
  20 #define NAND_ADDR0                      0x04
  21 #define NAND_ADDR1                      0x08
  22 #define NAND_FLASH_CHIP_SELECT          0x0c
  23 #define NAND_EXEC_CMD                   0x10
  24 #define NAND_FLASH_STATUS               0x14
  25 #define NAND_BUFFER_STATUS              0x18
  26 #define NAND_DEV0_CFG0                  0x20
  27 #define NAND_DEV0_CFG1                  0x24
  28 #define NAND_DEV0_ECC_CFG               0x28
  29 #define NAND_DEV1_ECC_CFG               0x2c
  30 #define NAND_DEV1_CFG0                  0x30
  31 #define NAND_DEV1_CFG1                  0x34
  32 #define NAND_READ_ID                    0x40
  33 #define NAND_READ_STATUS                0x44
  34 #define NAND_DEV_CMD0                   0xa0
  35 #define NAND_DEV_CMD1                   0xa4
  36 #define NAND_DEV_CMD2                   0xa8
  37 #define NAND_DEV_CMD_VLD                0xac
  38 #define SFLASHC_BURST_CFG               0xe0
  39 #define NAND_ERASED_CW_DETECT_CFG       0xe8
  40 #define NAND_ERASED_CW_DETECT_STATUS    0xec
  41 #define NAND_EBI2_ECC_BUF_CFG           0xf0
  42 #define FLASH_BUF_ACC                   0x100
  43
  44 #define NAND_CTRL                       0xf00
  45 #define NAND_VERSION                    0xf08
  46 #define NAND_READ_LOCATION_0            0xf20
  47 #define NAND_READ_LOCATION_1            0xf24
  48 #define NAND_READ_LOCATION_2            0xf28
  49 #define NAND_READ_LOCATION_3            0xf2c
  50
  51 /* dummy register offsets, used by write_reg_dma */
  52 #define NAND_DEV_CMD1_RESTORE           0xdead
  53 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  54
  55 /* NAND_FLASH_CMD bits */
  56 #define PAGE_ACC                        BIT(4)
  57 #define LAST_PAGE                       BIT(5)
  58
  59 /* NAND_FLASH_CHIP_SELECT bits */
  60 #define NAND_DEV_SEL                    0
  61 #define DM_EN                           BIT(2)
  62
  63 /* NAND_FLASH_STATUS bits */
  64 #define FS_OP_ERR                       BIT(4)
  65 #define FS_READY_BSY_N                  BIT(5)
  66 #define FS_MPU_ERR                      BIT(8)
  67 #define FS_DEVICE_STS_ERR               BIT(16)
  68 #define FS_DEVICE_WP                    BIT(23)
  69
  70 /* NAND_BUFFER_STATUS bits */
  71 #define BS_UNCORRECTABLE_BIT            BIT(8)
  72 #define BS_CORRECTABLE_ERR_MSK          0x1f
  73
  74 /* NAND_DEVn_CFG0 bits */
  75 #define DISABLE_STATUS_AFTER_WRITE      4
  76 #define CW_PER_PAGE                     6
  77 #define UD_SIZE_BYTES                   9
  78 #define ECC_PARITY_SIZE_BYTES_RS        19
  79 #define SPARE_SIZE_BYTES                23
  80 #define NUM_ADDR_CYCLES                 27
  81 #define STATUS_BFR_READ                 30
  82 #define SET_RD_MODE_AFTER_STATUS        31
  83
  84 /* NAND_DEVn_CFG0 bits */
  85 #define DEV0_CFG1_ECC_DISABLE           0
  86 #define WIDE_FLASH                      1
  87 #define NAND_RECOVERY_CYCLES            2
  88 #define CS_ACTIVE_BSY                   5
  89 #define BAD_BLOCK_BYTE_NUM              6
  90 #define BAD_BLOCK_IN_SPARE_AREA         16
  91 #define WR_RD_BSY_GAP                   17
  92 #define ENABLE_BCH_ECC                  27
  93
  94 /* NAND_DEV0_ECC_CFG bits */
  95 #define ECC_CFG_ECC_DISABLE             0
  96 #define ECC_SW_RESET                    1
  97 #define ECC_MODE                        4
  98 #define ECC_PARITY_SIZE_BYTES_BCH       8
  99 #define ECC_NUM_DATA_BYTES              16
 100 #define ECC_FORCE_CLK_OPEN              30
 101
 102 /* NAND_DEV_CMD1 bits */
 103 #define READ_ADDR                       0
 104
 105 /* NAND_DEV_CMD_VLD bits */
 106 #define READ_START_VLD                  BIT(0)
 107 #define READ_STOP_VLD                   BIT(1)
 108 #define WRITE_START_VLD                 BIT(2)
 109 #define ERASE_START_VLD                 BIT(3)
 110 #define SEQ_READ_START_VLD              BIT(4)
 111
 112 /* NAND_EBI2_ECC_BUF_CFG bits */
 113 #define NUM_STEPS                       0
 114
 115 /* NAND_ERASED_CW_DETECT_CFG bits */
 116 #define ERASED_CW_ECC_MASK              1
 117 #define AUTO_DETECT_RES                 0
 118 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 119 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 120 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 121 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 122 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 123
 124 /* NAND_ERASED_CW_DETECT_STATUS bits */
 125 #define PAGE_ALL_ERASED                 BIT(7)
 126 #define CODEWORD_ALL_ERASED             BIT(6)
 127 #define PAGE_ERASED                     BIT(5)
 128 #define CODEWORD_ERASED                 BIT(4)
 129 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 130 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 131
 132 /* NAND_READ_LOCATION_n bits */
 133 #define READ_LOCATION_OFFSET            0
 134 #define READ_LOCATION_SIZE              16
 135 #define READ_LOCATION_LAST              31
 136
 137 /* Version Mask */
 138 #define NAND_VERSION_MAJOR_MASK         0xf0000000
 139 #define NAND_VERSION_MAJOR_SHIFT        28
 140 #define NAND_VERSION_MINOR_MASK         0x0fff0000
 141 #define NAND_VERSION_MINOR_SHIFT        16
 142
 143 /* NAND OP_CMDs */
 144 #define OP_PAGE_READ                    0x2
 145 #define OP_PAGE_READ_WITH_ECC           0x3
 146 #define OP_PAGE_READ_WITH_ECC_SPARE     0x4
 147 #define OP_PROGRAM_PAGE                 0x6
 148 #define OP_PAGE_PROGRAM_WITH_ECC        0x7
 149 #define OP_PROGRAM_PAGE_SPARE           0x9
 150 #define OP_BLOCK_ERASE                  0xa
 151 #define OP_FETCH_ID                     0xb
 152 #define OP_RESET_DEVICE                 0xd
 153
 154 /* Default Value for NAND_DEV_CMD_VLD */
 155 #define NAND_DEV_CMD_VLD_VAL            (READ_START_VLD | WRITE_START_VLD | \
 156                                          ERASE_START_VLD | SEQ_READ_START_VLD)
 157
 158 /* NAND_CTRL bits */
 159 #define BAM_MODE_EN                     BIT(0)
 160
 161 /*
 162  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 163  * the driver calls the chunks 'step' or 'codeword' interchangeably
 164  */
 165 #define NANDC_STEP_SIZE                 512
 166
 167 /*
 168  * the largest page size we support is 8K, this will have 16 steps/codewords
 169  * of 512 bytes each
 170  */
 171 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 172
 173 /* we read at most 3 registers per codeword scan */
 174 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 175
 176 /* ECC modes supported by the controller */
 177 #define ECC_NONE        BIT(0)
 178 #define ECC_RS_4BIT     BIT(1)
 179 #define ECC_BCH_4BIT    BIT(2)
 180 #define ECC_BCH_8BIT    BIT(3)
 181
 182 #define nandc_set_read_loc(nandc, reg, offset, size, is_last)   \
 183 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg,                  \
 184               ((offset) << READ_LOCATION_OFFSET) |              \
 185               ((size) << READ_LOCATION_SIZE) |                  \
 186               ((is_last) << READ_LOCATION_LAST))
 187
 188 /*
 189  * Returns the actual register address for all NAND_DEV_ registers
 190  * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
 191  */
 192 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
 193
 194 /* Returns the NAND register physical address */
 195 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
 196
 197 /* Returns the dma address for reg read buffer */
 198 #define reg_buf_dma_addr(chip, vaddr) \
 199         ((chip)->reg_read_dma + \
 200         ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
 201
 202 #define QPIC_PER_CW_CMD_ELEMENTS        32
 203 #define QPIC_PER_CW_CMD_SGL             32
 204 #define QPIC_PER_CW_DATA_SGL            8
 205
 206 #define QPIC_NAND_COMPLETION_TIMEOUT    msecs_to_jiffies(2000)
 207
 208 /*
 209  * Flags used in DMA descriptor preparation helper functions
 210  * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
 211  */
 212 /* Don't set the EOT in current tx BAM sgl */
 213 #define NAND_BAM_NO_EOT                 BIT(0)
 214 /* Set the NWD flag in current BAM sgl */
 215 #define NAND_BAM_NWD                    BIT(1)
 216 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
 217 #define NAND_BAM_NEXT_SGL               BIT(2)
 218 /*
 219  * Erased codeword status is being used two times in single transfer so this
 220  * flag will determine the current value of erased codeword status register
 221  */
 222 #define NAND_ERASED_CW_SET              BIT(4)
 223
 224 /*
 225  * This data type corresponds to the BAM transaction which will be used for all
 226  * NAND transfers.
 227  * @bam_ce - the array of BAM command elements
 228  * @cmd_sgl - sgl for NAND BAM command pipe
 229  * @data_sgl - sgl for NAND BAM consumer/producer pipe
 230  * @bam_ce_pos - the index in bam_ce which is available for next sgl
 231  * @bam_ce_start - the index in bam_ce which marks the start position ce
 232  *                 for current sgl. It will be used for size calculation
 233  *                 for current sgl
 234  * @cmd_sgl_pos - current index in command sgl.
 235  * @cmd_sgl_start - start index in command sgl.
 236  * @tx_sgl_pos - current index in data sgl for tx.
 237  * @tx_sgl_start - start index in data sgl for tx.
 238  * @rx_sgl_pos - current index in data sgl for rx.
 239  * @rx_sgl_start - start index in data sgl for rx.
 240  * @wait_second_completion - wait for second DMA desc completion before making
 241  *                           the NAND transfer completion.
 242  * @txn_done - completion for NAND transfer.
 243  * @last_data_desc - last DMA desc in data channel (tx/rx).
 244  * @last_cmd_desc - last DMA desc in command channel.
 245  */
 246 struct bam_transaction {
 247         struct bam_cmd_element *bam_ce;
 248         struct scatterlist *cmd_sgl;
 249         struct scatterlist *data_sgl;
 250         u32 bam_ce_pos;
 251         u32 bam_ce_start;
 252         u32 cmd_sgl_pos;
 253         u32 cmd_sgl_start;
 254         u32 tx_sgl_pos;
 255         u32 tx_sgl_start;
 256         u32 rx_sgl_pos;
 257         u32 rx_sgl_start;
 258         bool wait_second_completion;
 259         struct completion txn_done;
 260         struct dma_async_tx_descriptor *last_data_desc;
 261         struct dma_async_tx_descriptor *last_cmd_desc;
 262 };
 263
 264 /*
 265  * This data type corresponds to the nand dma descriptor
 266  * @list - list for desc_info
 267  * @dir - DMA transfer direction
 268  * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
 269  *            ADM
 270  * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
 271  * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
 272  * @dma_desc - low level DMA engine descriptor
 273  */
 274 struct desc_info {
 275         struct list_head node;
 276
 277         enum dma_data_direction dir;
 278         union {
 279                 struct scatterlist adm_sgl;
 280                 struct {
 281                         struct scatterlist *bam_sgl;
 282                         int sgl_cnt;
 283                 };
 284         };
 285         struct dma_async_tx_descriptor *dma_desc;
 286 };
 287
 288 /*
 289  * holds the current register values that we want to write. acts as a contiguous
 290  * chunk of memory which we use to write the controller registers through DMA.
 291  */
 292 struct nandc_regs {
 293         __le32 cmd;
 294         __le32 addr0;
 295         __le32 addr1;
 296         __le32 chip_sel;
 297         __le32 exec;
 298
 299         __le32 cfg0;
 300         __le32 cfg1;
 301         __le32 ecc_bch_cfg;
 302
 303         __le32 clrflashstatus;
 304         __le32 clrreadstatus;
 305
 306         __le32 cmd1;
 307         __le32 vld;
 308
 309         __le32 orig_cmd1;
 310         __le32 orig_vld;
 311
 312         __le32 ecc_buf_cfg;
 313         __le32 read_location0;
 314         __le32 read_location1;
 315         __le32 read_location2;
 316         __le32 read_location3;
 317
 318         __le32 erased_cw_detect_cfg_clr;
 319         __le32 erased_cw_detect_cfg_set;
 320 };
 321
 322 /*
 323  * NAND controller data struct
 324  *
 325  * @controller:                 base controller structure
 326  * @host_list:                  list containing all the chips attached to the
 327  *                              controller
 328  * @dev:                        parent device
 329  * @base:                       MMIO base
 330  * @base_phys:                  physical base address of controller registers
 331  * @base_dma:                   dma base address of controller registers
 332  * @core_clk:                   controller clock
 333  * @aon_clk:                    another controller clock
 334  *
 335  * @chan:                       dma channel
 336  * @cmd_crci:                   ADM DMA CRCI for command flow control
 337  * @data_crci:                  ADM DMA CRCI for data flow control
 338  * @desc_list:                  DMA descriptor list (list of desc_infos)
 339  *
 340  * @data_buffer:                our local DMA buffer for page read/writes,
 341  *                              used when we can't use the buffer provided
 342  *                              by upper layers directly
 343  * @buf_size/count/start:       markers for chip->legacy.read_buf/write_buf
 344  *                              functions
 345  * @reg_read_buf:               local buffer for reading back registers via DMA
 346  * @reg_read_dma:               contains dma address for register read buffer
 347  * @reg_read_pos:               marker for data read in reg_read_buf
 348  *
 349  * @regs:                       a contiguous chunk of memory for DMA register
 350  *                              writes. contains the register values to be
 351  *                              written to controller
 352  * @cmd1/vld:                   some fixed controller register values
 353  * @props:                      properties of current NAND controller,
 354  *                              initialized via DT match data
 355  * @max_cwperpage:              maximum QPIC codewords required. calculated
 356  *                              from all connected NAND devices pagesize
 357  */
 358 struct qcom_nand_controller {
 359         struct nand_controller controller;
 360         struct list_head host_list;
 361
 362         struct device *dev;
 363
 364         void __iomem *base;
 365         phys_addr_t base_phys;
 366         dma_addr_t base_dma;
 367
 368         struct clk *core_clk;
 369         struct clk *aon_clk;
 370
 371         union {
 372                 /* will be used only by QPIC for BAM DMA */
 373                 struct {
 374                         struct dma_chan *tx_chan;
 375                         struct dma_chan *rx_chan;
 376                         struct dma_chan *cmd_chan;
 377                 };
 378
 379                 /* will be used only by EBI2 for ADM DMA */
 380                 struct {
 381                         struct dma_chan *chan;
 382                         unsigned int cmd_crci;
 383                         unsigned int data_crci;
 384                 };
 385         };
 386
 387         struct list_head desc_list;
 388         struct bam_transaction *bam_txn;
 389
 390         u8              *data_buffer;
 391         int             buf_size;
 392         int             buf_count;
 393         int             buf_start;
 394         unsigned int    max_cwperpage;
 395
 396         __le32 *reg_read_buf;
 397         dma_addr_t reg_read_dma;
 398         int reg_read_pos;
 399
 400         struct nandc_regs *regs;
 401
 402         u32 cmd1, vld;
 403         const struct qcom_nandc_props *props;
 404 };
 405
 406 /*
 407  * NAND chip structure
 408  *
 409  * @chip:                       base NAND chip structure
 410  * @node:                       list node to add itself to host_list in
 411  *                              qcom_nand_controller
 412  *
 413  * @cs:                         chip select value for this chip
 414  * @cw_size:                    the number of bytes in a single step/codeword
 415  *                              of a page, consisting of all data, ecc, spare
 416  *                              and reserved bytes
 417  * @cw_data:                    the number of bytes within a codeword protected
 418  *                              by ECC
 419  * @use_ecc:                    request the controller to use ECC for the
 420  *                              upcoming read/write
 421  * @bch_enabled:                flag to tell whether BCH ECC mode is used
 422  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 423  *                              chip
 424  * @status:                     value to be returned if NAND_CMD_STATUS command
 425  *                              is executed
 426  * @last_command:               keeps track of last command on this chip. used
 427  *                              for reading correct status
 428  *
 429  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 430  *                              ecc/non-ecc mode for the current nand flash
 431  *                              device
 432  */
 433 struct qcom_nand_host {
 434         struct nand_chip chip;
 435         struct list_head node;
 436
 437         int cs;
 438         int cw_size;
 439         int cw_data;
 440         bool use_ecc;
 441         bool bch_enabled;
 442         int ecc_bytes_hw;
 443         int spare_bytes;
 444         int bbm_size;
 445         u8 status;
 446         int last_command;
 447
 448         u32 cfg0, cfg1;
 449         u32 cfg0_raw, cfg1_raw;
 450         u32 ecc_buf_cfg;
 451         u32 ecc_bch_cfg;
 452         u32 clrflashstatus;
 453         u32 clrreadstatus;
 454 };
 455
 456 /*
 457  * This data type corresponds to the NAND controller properties which varies
 458  * among different NAND controllers.
 459  * @ecc_modes - ecc mode for NAND
 460  * @is_bam - whether NAND controller is using BAM
 461  * @is_qpic - whether NAND CTRL is part of qpic IP
 462  * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
 463  */
 464 struct qcom_nandc_props {
 465         u32 ecc_modes;
 466         bool is_bam;
 467         bool is_qpic;
 468         u32 dev_cmd_reg_start;
 469 };
 470
 471 /* Frees the BAM transaction memory */
 472 static void free_bam_transaction(struct qcom_nand_controller *nandc)
 473 {
 474         struct bam_transaction *bam_txn = nandc->bam_txn;
 475
 476         devm_kfree(nandc->dev, bam_txn);
 477 }
 478
 479 /* Allocates and Initializes the BAM transaction */
 480 static struct bam_transaction *
 481 alloc_bam_transaction(struct qcom_nand_controller *nandc)
 482 {
 483         struct bam_transaction *bam_txn;
 484         size_t bam_txn_size;
 485         unsigned int num_cw = nandc->max_cwperpage;
 486         void *bam_txn_buf;
 487
 488         bam_txn_size =
 489                 sizeof(*bam_txn) + num_cw *
 490                 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
 491                 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
 492                 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
 493
 494         bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
 495         if (!bam_txn_buf)
 496                 return NULL;
 497
 498         bam_txn = bam_txn_buf;
 499         bam_txn_buf += sizeof(*bam_txn);
 500
 501         bam_txn->bam_ce = bam_txn_buf;
 502         bam_txn_buf +=
 503                 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
 504
 505         bam_txn->cmd_sgl = bam_txn_buf;
 506         bam_txn_buf +=
 507                 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
 508
 509         bam_txn->data_sgl = bam_txn_buf;
 510
 511         init_completion(&bam_txn->txn_done);
 512
 513         return bam_txn;
 514 }
 515
 516 /* Clears the BAM transaction indexes */
 517 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
 518 {
 519         struct bam_transaction *bam_txn = nandc->bam_txn;
 520
 521         if (!nandc->props->is_bam)
 522                 return;
 523
 524         bam_txn->bam_ce_pos = 0;
 525         bam_txn->bam_ce_start = 0;
 526         bam_txn->cmd_sgl_pos = 0;
 527         bam_txn->cmd_sgl_start = 0;
 528         bam_txn->tx_sgl_pos = 0;
 529         bam_txn->tx_sgl_start = 0;
 530         bam_txn->rx_sgl_pos = 0;
 531         bam_txn->rx_sgl_start = 0;
 532         bam_txn->last_data_desc = NULL;
 533         bam_txn->wait_second_completion = false;
 534
 535         sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
 536                       QPIC_PER_CW_CMD_SGL);
 537         sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
 538                       QPIC_PER_CW_DATA_SGL);
 539
 540         reinit_completion(&bam_txn->txn_done);
 541 }
 542
 543 /* Callback for DMA descriptor completion */
 544 static void qpic_bam_dma_done(void *data)
 545 {
 546         struct bam_transaction *bam_txn = data;
 547
 548         /*
 549          * In case of data transfer with NAND, 2 callbacks will be generated.
 550          * One for command channel and another one for data channel.
 551          * If current transaction has data descriptors
 552          * (i.e. wait_second_completion is true), then set this to false
 553          * and wait for second DMA descriptor completion.
 554          */
 555         if (bam_txn->wait_second_completion)
 556                 bam_txn->wait_second_completion = false;
 557         else
 558                 complete(&bam_txn->txn_done);
 559 }
 560
 561 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 562 {
 563         return container_of(chip, struct qcom_nand_host, chip);
 564 }
 565
 566 static inline struct qcom_nand_controller *
 567 get_qcom_nand_controller(struct nand_chip *chip)
 568 {
 569         return container_of(chip->controller, struct qcom_nand_controller,
 570                             controller);
 571 }
 572
 573 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 574 {
 575         return ioread32(nandc->base + offset);
 576 }
 577
 578 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 579                                u32 val)
 580 {
 581         iowrite32(val, nandc->base + offset);
 582 }
 583
 584 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
 585                                           bool is_cpu)
 586 {
 587         if (!nandc->props->is_bam)
 588                 return;
 589
 590         if (is_cpu)
 591                 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
 592                                         MAX_REG_RD *
 593                                         sizeof(*nandc->reg_read_buf),
 594                                         DMA_FROM_DEVICE);
 595         else
 596                 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
 597                                            MAX_REG_RD *
 598                                            sizeof(*nandc->reg_read_buf),
 599                                            DMA_FROM_DEVICE);
 600 }
 601
 602 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 603 {
 604         switch (offset) {
 605         case NAND_FLASH_CMD:
 606                 return &regs->cmd;
 607         case NAND_ADDR0:
 608                 return &regs->addr0;
 609         case NAND_ADDR1:
 610                 return &regs->addr1;
 611         case NAND_FLASH_CHIP_SELECT:
 612                 return &regs->chip_sel;
 613         case NAND_EXEC_CMD:
 614                 return &regs->exec;
 615         case NAND_FLASH_STATUS:
 616                 return &regs->clrflashstatus;
 617         case NAND_DEV0_CFG0:
 618                 return &regs->cfg0;
 619         case NAND_DEV0_CFG1:
 620                 return &regs->cfg1;
 621         case NAND_DEV0_ECC_CFG:
 622                 return &regs->ecc_bch_cfg;
 623         case NAND_READ_STATUS:
 624                 return &regs->clrreadstatus;
 625         case NAND_DEV_CMD1:
 626                 return &regs->cmd1;
 627         case NAND_DEV_CMD1_RESTORE:
 628                 return &regs->orig_cmd1;
 629         case NAND_DEV_CMD_VLD:
 630                 return &regs->vld;
 631         case NAND_DEV_CMD_VLD_RESTORE:
 632                 return &regs->orig_vld;
 633         case NAND_EBI2_ECC_BUF_CFG:
 634                 return &regs->ecc_buf_cfg;
 635         case NAND_READ_LOCATION_0:
 636                 return &regs->read_location0;
 637         case NAND_READ_LOCATION_1:
 638                 return &regs->read_location1;
 639         case NAND_READ_LOCATION_2:
 640                 return &regs->read_location2;
 641         case NAND_READ_LOCATION_3:
 642                 return &regs->read_location3;
 643         default:
 644                 return NULL;
 645         }
 646 }
 647
 648 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
 649                           u32 val)
 650 {
 651         struct nandc_regs *regs = nandc->regs;
 652         __le32 *reg;
 653
 654         reg = offset_to_nandc_reg(regs, offset);
 655
 656         if (reg)
 657                 *reg = cpu_to_le32(val);
 658 }
 659
 660 /* helper to configure address register values */
 661 static void set_address(struct qcom_nand_host *host, u16 column, int page)
 662 {
 663         struct nand_chip *chip = &host->chip;
 664         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 665
 666         if (chip->options & NAND_BUSWIDTH_16)
 667                 column >>= 1;
 668
 669         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
 670         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
 671 }
 672
 673 /*
 674  * update_rw_regs:      set up read/write register values, these will be
 675  *                      written to the NAND controller registers via DMA
 676  *
 677  * @num_cw:             number of steps for the read/write operation
 678  * @read:               read or write operation
 679  */
 680 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 681 {
 682         struct nand_chip *chip = &host->chip;
 683         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 684         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 685
 686         if (read) {
 687                 if (host->use_ecc)
 688                         cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 689                 else
 690                         cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
 691         } else {
 692                 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 693         }
 694
 695         if (host->use_ecc) {
 696                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 697                                 (num_cw - 1) << CW_PER_PAGE;
 698
 699                 cfg1 = host->cfg1;
 700                 ecc_bch_cfg = host->ecc_bch_cfg;
 701         } else {
 702                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 703                                 (num_cw - 1) << CW_PER_PAGE;
 704
 705                 cfg1 = host->cfg1_raw;
 706                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 707         }
 708
 709         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
 710         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
 711         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
 712         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 713         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 714         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 715         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 716         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 717
 718         if (read)
 719                 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
 720                                    host->cw_data : host->cw_size, 1);
 721 }
 722
 723 /*
 724  * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
 725  * for BAM. This descriptor will be added in the NAND DMA descriptor queue
 726  * which will be submitted to DMA engine.
 727  */
 728 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
 729                                   struct dma_chan *chan,
 730                                   unsigned long flags)
 731 {
 732         struct desc_info *desc;
 733         struct scatterlist *sgl;
 734         unsigned int sgl_cnt;
 735         int ret;
 736         struct bam_transaction *bam_txn = nandc->bam_txn;
 737         enum dma_transfer_direction dir_eng;
 738         struct dma_async_tx_descriptor *dma_desc;
 739
 740         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 741         if (!desc)
 742                 return -ENOMEM;
 743
 744         if (chan == nandc->cmd_chan) {
 745                 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
 746                 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
 747                 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
 748                 dir_eng = DMA_MEM_TO_DEV;
 749                 desc->dir = DMA_TO_DEVICE;
 750         } else if (chan == nandc->tx_chan) {
 751                 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
 752                 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
 753                 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
 754                 dir_eng = DMA_MEM_TO_DEV;
 755                 desc->dir = DMA_TO_DEVICE;
 756         } else {
 757                 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
 758                 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
 759                 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
 760                 dir_eng = DMA_DEV_TO_MEM;
 761                 desc->dir = DMA_FROM_DEVICE;
 762         }
 763
 764         sg_mark_end(sgl + sgl_cnt - 1);
 765         ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 766         if (ret == 0) {
 767                 dev_err(nandc->dev, "failure in mapping desc\n");
 768                 kfree(desc);
 769                 return -ENOMEM;
 770         }
 771
 772         desc->sgl_cnt = sgl_cnt;
 773         desc->bam_sgl = sgl;
 774
 775         dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
 776                                            flags);
 777
 778         if (!dma_desc) {
 779                 dev_err(nandc->dev, "failure in prep desc\n");
 780                 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 781                 kfree(desc);
 782                 return -EINVAL;
 783         }
 784
 785         desc->dma_desc = dma_desc;
 786
 787         /* update last data/command descriptor */
 788         if (chan == nandc->cmd_chan)
 789                 bam_txn->last_cmd_desc = dma_desc;
 790         else
 791                 bam_txn->last_data_desc = dma_desc;
 792
 793         list_add_tail(&desc->node, &nandc->desc_list);
 794
 795         return 0;
 796 }
 797
 798 /*
 799  * Prepares the command descriptor for BAM DMA which will be used for NAND
 800  * register reads and writes. The command descriptor requires the command
 801  * to be formed in command element type so this function uses the command
 802  * element from bam transaction ce array and fills the same with required
 803  * data. A single SGL can contain multiple command elements so
 804  * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
 805  * after the current command element.
 806  */
 807 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
 808                                  int reg_off, const void *vaddr,
 809                                  int size, unsigned int flags)
 810 {
 811         int bam_ce_size;
 812         int i, ret;
 813         struct bam_cmd_element *bam_ce_buffer;
 814         struct bam_transaction *bam_txn = nandc->bam_txn;
 815
 816         bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
 817
 818         /* fill the command desc */
 819         for (i = 0; i < size; i++) {
 820                 if (read)
 821                         bam_prep_ce(&bam_ce_buffer[i],
 822                                     nandc_reg_phys(nandc, reg_off + 4 * i),
 823                                     BAM_READ_COMMAND,
 824                                     reg_buf_dma_addr(nandc,
 825                                                      (__le32 *)vaddr + i));
 826                 else
 827                         bam_prep_ce_le32(&bam_ce_buffer[i],
 828                                          nandc_reg_phys(nandc, reg_off + 4 * i),
 829                                          BAM_WRITE_COMMAND,
 830                                          *((__le32 *)vaddr + i));
 831         }
 832
 833         bam_txn->bam_ce_pos += size;
 834
 835         /* use the separate sgl after this command */
 836         if (flags & NAND_BAM_NEXT_SGL) {
 837                 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
 838                 bam_ce_size = (bam_txn->bam_ce_pos -
 839                                 bam_txn->bam_ce_start) *
 840                                 sizeof(struct bam_cmd_element);
 841                 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
 842                            bam_ce_buffer, bam_ce_size);
 843                 bam_txn->cmd_sgl_pos++;
 844                 bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
 845
 846                 if (flags & NAND_BAM_NWD) {
 847                         ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
 848                                                      DMA_PREP_FENCE |
 849                                                      DMA_PREP_CMD);
 850                         if (ret)
 851                                 return ret;
 852                 }
 853         }
 854
 855         return 0;
 856 }
 857
 858 /*
 859  * Prepares the data descriptor for BAM DMA which will be used for NAND
 860  * data reads and writes.
 861  */
 862 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
 863                                   const void *vaddr,
 864                                   int size, unsigned int flags)
 865 {
 866         int ret;
 867         struct bam_transaction *bam_txn = nandc->bam_txn;
 868
 869         if (read) {
 870                 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
 871                            vaddr, size);
 872                 bam_txn->rx_sgl_pos++;
 873         } else {
 874                 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
 875                            vaddr, size);
 876                 bam_txn->tx_sgl_pos++;
 877
 878                 /*
 879                  * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
 880                  * is not set, form the DMA descriptor
 881                  */
 882                 if (!(flags & NAND_BAM_NO_EOT)) {
 883                         ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
 884                                                      DMA_PREP_INTERRUPT);
 885                         if (ret)
 886                                 return ret;
 887                 }
 888         }
 889
 890         return 0;
 891 }
 892
 893 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
 894                              int reg_off, const void *vaddr, int size,
 895                              bool flow_control)
 896 {
 897         struct desc_info *desc;
 898         struct dma_async_tx_descriptor *dma_desc;
 899         struct scatterlist *sgl;
 900         struct dma_slave_config slave_conf;
 901         enum dma_transfer_direction dir_eng;
 902         int ret;
 903
 904         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 905         if (!desc)
 906                 return -ENOMEM;
 907
 908         sgl = &desc->adm_sgl;
 909
 910         sg_init_one(sgl, vaddr, size);
 911
 912         if (read) {
 913                 dir_eng = DMA_DEV_TO_MEM;
 914                 desc->dir = DMA_FROM_DEVICE;
 915         } else {
 916                 dir_eng = DMA_MEM_TO_DEV;
 917                 desc->dir = DMA_TO_DEVICE;
 918         }
 919
 920         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 921         if (ret == 0) {
 922                 ret = -ENOMEM;
 923                 goto err;
 924         }
 925
 926         memset(&slave_conf, 0x00, sizeof(slave_conf));
 927
 928         slave_conf.device_fc = flow_control;
 929         if (read) {
 930                 slave_conf.src_maxburst = 16;
 931                 slave_conf.src_addr = nandc->base_dma + reg_off;
 932                 slave_conf.slave_id = nandc->data_crci;
 933         } else {
 934                 slave_conf.dst_maxburst = 16;
 935                 slave_conf.dst_addr = nandc->base_dma + reg_off;
 936                 slave_conf.slave_id = nandc->cmd_crci;
 937         }
 938
 939         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 940         if (ret) {
 941                 dev_err(nandc->dev, "failed to configure dma channel\n");
 942                 goto err;
 943         }
 944
 945         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
 946         if (!dma_desc) {
 947                 dev_err(nandc->dev, "failed to prepare desc\n");
 948                 ret = -EINVAL;
 949                 goto err;
 950         }
 951
 952         desc->dma_desc = dma_desc;
 953
 954         list_add_tail(&desc->node, &nandc->desc_list);
 955
 956         return 0;
 957 err:
 958         kfree(desc);
 959
 960         return ret;
 961 }
 962
 963 /*
 964  * read_reg_dma:        prepares a descriptor to read a given number of
 965  *                      contiguous registers to the reg_read_buf pointer
 966  *
 967  * @first:              offset of the first register in the contiguous block
 968  * @num_regs:           number of registers to read
 969  * @flags:              flags to control DMA descriptor preparation
 970  */
 971 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 972                         int num_regs, unsigned int flags)
 973 {
 974         bool flow_control = false;
 975         void *vaddr;
 976
 977         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 978         nandc->reg_read_pos += num_regs;
 979
 980         if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
 981                 first = dev_cmd_reg_addr(nandc, first);
 982
 983         if (nandc->props->is_bam)
 984                 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
 985                                              num_regs, flags);
 986
 987         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 988                 flow_control = true;
 989
 990         return prep_adm_dma_desc(nandc, true, first, vaddr,
 991                                  num_regs * sizeof(u32), flow_control);
 992 }
 993
 994 /*
 995  * write_reg_dma:       prepares a descriptor to write a given number of
 996  *                      contiguous registers
 997  *
 998  * @first:              offset of the first register in the contiguous block
 999  * @num_regs:           number of registers to write
1000  * @flags:              flags to control DMA descriptor preparation
1001  */
1002 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1003                          int num_regs, unsigned int flags)
1004 {
1005         bool flow_control = false;
1006         struct nandc_regs *regs = nandc->regs;
1007         void *vaddr;
1008
1009         vaddr = offset_to_nandc_reg(regs, first);
1010
1011         if (first == NAND_ERASED_CW_DETECT_CFG) {
1012                 if (flags & NAND_ERASED_CW_SET)
1013                         vaddr = &regs->erased_cw_detect_cfg_set;
1014                 else
1015                         vaddr = &regs->erased_cw_detect_cfg_clr;
1016         }
1017
1018         if (first == NAND_EXEC_CMD)
1019                 flags |= NAND_BAM_NWD;
1020
1021         if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1022                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1023
1024         if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1025                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1026
1027         if (nandc->props->is_bam)
1028                 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1029                                              num_regs, flags);
1030
1031         if (first == NAND_FLASH_CMD)
1032                 flow_control = true;
1033
1034         return prep_adm_dma_desc(nandc, false, first, vaddr,
1035                                  num_regs * sizeof(u32), flow_control);
1036 }
1037
1038 /*
1039  * read_data_dma:       prepares a DMA descriptor to transfer data from the
1040  *                      controller's internal buffer to the buffer 'vaddr'
1041  *
1042  * @reg_off:            offset within the controller's data buffer
1043  * @vaddr:              virtual address of the buffer we want to write to
1044  * @size:               DMA transaction size in bytes
1045  * @flags:              flags to control DMA descriptor preparation
1046  */
1047 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1048                          const u8 *vaddr, int size, unsigned int flags)
1049 {
1050         if (nandc->props->is_bam)
1051                 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1052
1053         return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1054 }
1055
1056 /*
1057  * write_data_dma:      prepares a DMA descriptor to transfer data from
1058  *                      'vaddr' to the controller's internal buffer
1059  *
1060  * @reg_off:            offset within the controller's data buffer
1061  * @vaddr:              virtual address of the buffer we want to read from
1062  * @size:               DMA transaction size in bytes
1063  * @flags:              flags to control DMA descriptor preparation
1064  */
1065 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1066                           const u8 *vaddr, int size, unsigned int flags)
1067 {
1068         if (nandc->props->is_bam)
1069                 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1070
1071         return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1072 }
1073
1074 /*
1075  * Helper to prepare DMA descriptors for configuring registers
1076  * before reading a NAND page.
1077  */
1078 static void config_nand_page_read(struct qcom_nand_controller *nandc)
1079 {
1080         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1081         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1082         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1083         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1084         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1085                       NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1086 }
1087
1088 /*
1089  * Helper to prepare DMA descriptors for configuring registers
1090  * before reading each codeword in NAND page.
1091  */
1092 static void
1093 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc)
1094 {
1095         if (nandc->props->is_bam)
1096                 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
1097                               NAND_BAM_NEXT_SGL);
1098
1099         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1100         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1101
1102         if (use_ecc) {
1103                 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1104                 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1105                              NAND_BAM_NEXT_SGL);
1106         } else {
1107                 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1108         }
1109 }
1110
1111 /*
1112  * Helper to prepare dma descriptors to configure registers needed for reading a
1113  * single codeword in page
1114  */
1115 static void
1116 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc,
1117                                 bool use_ecc)
1118 {
1119         config_nand_page_read(nandc);
1120         config_nand_cw_read(nandc, use_ecc);
1121 }
1122
1123 /*
1124  * Helper to prepare DMA descriptors used to configure registers needed for
1125  * before writing a NAND page.
1126  */
1127 static void config_nand_page_write(struct qcom_nand_controller *nandc)
1128 {
1129         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1130         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1131         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1132                       NAND_BAM_NEXT_SGL);
1133 }
1134
1135 /*
1136  * Helper to prepare DMA descriptors for configuring registers
1137  * before writing each codeword in NAND page.
1138  */
1139 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1140 {
1141         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1142         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1143
1144         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1145
1146         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1147         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1148 }
1149
1150 /*
1151  * the following functions are used within chip->legacy.cmdfunc() to
1152  * perform different NAND_CMD_* commands
1153  */
1154
1155 /* sets up descriptors for NAND_CMD_PARAM */
1156 static int nandc_param(struct qcom_nand_host *host)
1157 {
1158         struct nand_chip *chip = &host->chip;
1159         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1160
1161         /*
1162          * NAND_CMD_PARAM is called before we know much about the FLASH chip
1163          * in use. we configure the controller to perform a raw read of 512
1164          * bytes to read onfi params
1165          */
1166         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1167         nandc_set_reg(nandc, NAND_ADDR0, 0);
1168         nandc_set_reg(nandc, NAND_ADDR1, 0);
1169         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1170                                         | 512 << UD_SIZE_BYTES
1171                                         | 5 << NUM_ADDR_CYCLES
1172                                         | 0 << SPARE_SIZE_BYTES);
1173         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1174                                         | 0 << CS_ACTIVE_BSY
1175                                         | 17 << BAD_BLOCK_BYTE_NUM
1176                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
1177                                         | 2 << WR_RD_BSY_GAP
1178                                         | 0 << WIDE_FLASH
1179                                         | 1 << DEV0_CFG1_ECC_DISABLE);
1180         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1181
1182         /* configure CMD1 and VLD for ONFI param probing */
1183         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1184                       (nandc->vld & ~READ_START_VLD));
1185         nandc_set_reg(nandc, NAND_DEV_CMD1,
1186                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
1187                       | NAND_CMD_PARAM << READ_ADDR);
1188
1189         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1190
1191         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1192         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1193         nandc_set_read_loc(nandc, 0, 0, 512, 1);
1194
1195         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1196         write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1197
1198         nandc->buf_count = 512;
1199         memset(nandc->data_buffer, 0xff, nandc->buf_count);
1200
1201         config_nand_single_cw_page_read(nandc, false);
1202
1203         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1204                       nandc->buf_count, 0);
1205
1206         /* restore CMD1 and VLD regs */
1207         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1208         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1209
1210         return 0;
1211 }
1212
1213 /* sets up descriptors for NAND_CMD_ERASE1 */
1214 static int erase_block(struct qcom_nand_host *host, int page_addr)
1215 {
1216         struct nand_chip *chip = &host->chip;
1217         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1218
1219         nandc_set_reg(nandc, NAND_FLASH_CMD,
1220                       OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1221         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1222         nandc_set_reg(nandc, NAND_ADDR1, 0);
1223         nandc_set_reg(nandc, NAND_DEV0_CFG0,
1224                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
1225         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1226         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1227         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1228         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1229
1230         write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1231         write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1232         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1233
1234         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1235
1236         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1237         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1238
1239         return 0;
1240 }
1241
1242 /* sets up descriptors for NAND_CMD_READID */
1243 static int read_id(struct qcom_nand_host *host, int column)
1244 {
1245         struct nand_chip *chip = &host->chip;
1246         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1247
1248         if (column == -1)
1249                 return 0;
1250
1251         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1252         nandc_set_reg(nandc, NAND_ADDR0, column);
1253         nandc_set_reg(nandc, NAND_ADDR1, 0);
1254         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1255                       nandc->props->is_bam ? 0 : DM_EN);
1256         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1257
1258         write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1259         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1260
1261         read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1262
1263         return 0;
1264 }
1265
1266 /* sets up descriptors for NAND_CMD_RESET */
1267 static int reset(struct qcom_nand_host *host)
1268 {
1269         struct nand_chip *chip = &host->chip;
1270         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1271
1272         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1273         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1274
1275         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1276         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1277
1278         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1279
1280         return 0;
1281 }
1282
1283 /* helpers to submit/free our list of dma descriptors */
1284 static int submit_descs(struct qcom_nand_controller *nandc)
1285 {
1286         struct desc_info *desc;
1287         dma_cookie_t cookie = 0;
1288         struct bam_transaction *bam_txn = nandc->bam_txn;
1289         int r;
1290
1291         if (nandc->props->is_bam) {
1292                 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1293                         r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1294                         if (r)
1295                                 return r;
1296                 }
1297
1298                 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1299                         r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1300                                                    DMA_PREP_INTERRUPT);
1301                         if (r)
1302                                 return r;
1303                 }
1304
1305                 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1306                         r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1307                                                    DMA_PREP_CMD);
1308                         if (r)
1309                                 return r;
1310                 }
1311         }
1312
1313         list_for_each_entry(desc, &nandc->desc_list, node)
1314                 cookie = dmaengine_submit(desc->dma_desc);
1315
1316         if (nandc->props->is_bam) {
1317                 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1318                 bam_txn->last_cmd_desc->callback_param = bam_txn;
1319                 if (bam_txn->last_data_desc) {
1320                         bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1321                         bam_txn->last_data_desc->callback_param = bam_txn;
1322                         bam_txn->wait_second_completion = true;
1323                 }
1324
1325                 dma_async_issue_pending(nandc->tx_chan);
1326                 dma_async_issue_pending(nandc->rx_chan);
1327                 dma_async_issue_pending(nandc->cmd_chan);
1328
1329                 if (!wait_for_completion_timeout(&bam_txn->txn_done,
1330                                                  QPIC_NAND_COMPLETION_TIMEOUT))
1331                         return -ETIMEDOUT;
1332         } else {
1333                 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1334                         return -ETIMEDOUT;
1335         }
1336
1337         return 0;
1338 }
1339
1340 static void free_descs(struct qcom_nand_controller *nandc)
1341 {
1342         struct desc_info *desc, *n;
1343
1344         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1345                 list_del(&desc->node);
1346
1347                 if (nandc->props->is_bam)
1348                         dma_unmap_sg(nandc->dev, desc->bam_sgl,
1349                                      desc->sgl_cnt, desc->dir);
1350                 else
1351                         dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1352                                      desc->dir);
1353
1354                 kfree(desc);
1355         }
1356 }
1357
1358 /* reset the register read buffer for next NAND operation */
1359 static void clear_read_regs(struct qcom_nand_controller *nandc)
1360 {
1361         nandc->reg_read_pos = 0;
1362         nandc_read_buffer_sync(nandc, false);
1363 }
1364
1365 static void pre_command(struct qcom_nand_host *host, int command)
1366 {
1367         struct nand_chip *chip = &host->chip;
1368         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1369
1370         nandc->buf_count = 0;
1371         nandc->buf_start = 0;
1372         host->use_ecc = false;
1373         host->last_command = command;
1374
1375         clear_read_regs(nandc);
1376
1377         if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1378             command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1379                 clear_bam_transaction(nandc);
1380 }
1381
1382 /*
1383  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1384  * privately maintained status byte, this status byte can be read after
1385  * NAND_CMD_STATUS is called
1386  */
1387 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1388 {
1389         struct nand_chip *chip = &host->chip;
1390         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1391         struct nand_ecc_ctrl *ecc = &chip->ecc;
1392         int num_cw;
1393         int i;
1394
1395         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1396         nandc_read_buffer_sync(nandc, true);
1397
1398         for (i = 0; i < num_cw; i++) {
1399                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1400
1401                 if (flash_status & FS_MPU_ERR)
1402                         host->status &= ~NAND_STATUS_WP;
1403
1404                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1405                                                  (flash_status &
1406                                                   FS_DEVICE_STS_ERR)))
1407                         host->status |= NAND_STATUS_FAIL;
1408         }
1409 }
1410
1411 static void post_command(struct qcom_nand_host *host, int command)
1412 {
1413         struct nand_chip *chip = &host->chip;
1414         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1415
1416         switch (command) {
1417         case NAND_CMD_READID:
1418                 nandc_read_buffer_sync(nandc, true);
1419                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1420                        nandc->buf_count);
1421                 break;
1422         case NAND_CMD_PAGEPROG:
1423         case NAND_CMD_ERASE1:
1424                 parse_erase_write_errors(host, command);
1425                 break;
1426         default:
1427                 break;
1428         }
1429 }
1430
1431 /*
1432  * Implements chip->legacy.cmdfunc. It's  only used for a limited set of
1433  * commands. The rest of the commands wouldn't be called by upper layers.
1434  * For example, NAND_CMD_READOOB would never be called because we have our own
1435  * versions of read_oob ops for nand_ecc_ctrl.
1436  */
1437 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command,
1438                                int column, int page_addr)
1439 {
1440         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1441         struct nand_ecc_ctrl *ecc = &chip->ecc;
1442         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1443         bool wait = false;
1444         int ret = 0;
1445
1446         pre_command(host, command);
1447
1448         switch (command) {
1449         case NAND_CMD_RESET:
1450                 ret = reset(host);
1451                 wait = true;
1452                 break;
1453
1454         case NAND_CMD_READID:
1455                 nandc->buf_count = 4;
1456                 ret = read_id(host, column);
1457                 wait = true;
1458                 break;
1459
1460         case NAND_CMD_PARAM:
1461                 ret = nandc_param(host);
1462                 wait = true;
1463                 break;
1464
1465         case NAND_CMD_ERASE1:
1466                 ret = erase_block(host, page_addr);
1467                 wait = true;
1468                 break;
1469
1470         case NAND_CMD_READ0:
1471                 /* we read the entire page for now */
1472                 WARN_ON(column != 0);
1473
1474                 host->use_ecc = true;
1475                 set_address(host, 0, page_addr);
1476                 update_rw_regs(host, ecc->steps, true);
1477                 break;
1478
1479         case NAND_CMD_SEQIN:
1480                 WARN_ON(column != 0);
1481                 set_address(host, 0, page_addr);
1482                 break;
1483
1484         case NAND_CMD_PAGEPROG:
1485         case NAND_CMD_STATUS:
1486         case NAND_CMD_NONE:
1487         default:
1488                 break;
1489         }
1490
1491         if (ret) {
1492                 dev_err(nandc->dev, "failure executing command %d\n",
1493                         command);
1494                 free_descs(nandc);
1495                 return;
1496         }
1497
1498         if (wait) {
1499                 ret = submit_descs(nandc);
1500                 if (ret)
1501                         dev_err(nandc->dev,
1502                                 "failure submitting descs for command %d\n",
1503                                 command);
1504         }
1505
1506         free_descs(nandc);
1507
1508         post_command(host, command);
1509 }
1510
1511 /*
1512  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1513  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1514  *
1515  * when using RS ECC, the HW reports the same erros when reading an erased CW,
1516  * but it notifies that it is an erased CW by placing special characters at
1517  * certain offsets in the buffer.
1518  *
1519  * verify if the page is erased or not, and fix up the page for RS ECC by
1520  * replacing the special characters with 0xff.
1521  */
1522 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1523 {
1524         u8 empty1, empty2;
1525
1526         /*
1527          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1528          * is erased by looking for 0x54s at offsets 3 and 175 from the
1529          * beginning of each codeword
1530          */
1531
1532         empty1 = data_buf[3];
1533         empty2 = data_buf[175];
1534
1535         /*
1536          * if the erased codework markers, if they exist override them with
1537          * 0xffs
1538          */
1539         if ((empty1 == 0x54 && empty2 == 0xff) ||
1540             (empty1 == 0xff && empty2 == 0x54)) {
1541                 data_buf[3] = 0xff;
1542                 data_buf[175] = 0xff;
1543         }
1544
1545         /*
1546          * check if the entire chunk contains 0xffs or not. if it doesn't, then
1547          * restore the original values at the special offsets
1548          */
1549         if (memchr_inv(data_buf, 0xff, data_len)) {
1550                 data_buf[3] = empty1;
1551                 data_buf[175] = empty2;
1552
1553                 return false;
1554         }
1555
1556         return true;
1557 }
1558
1559 struct read_stats {
1560         __le32 flash;
1561         __le32 buffer;
1562         __le32 erased_cw;
1563 };
1564
1565 /* reads back FLASH_STATUS register set by the controller */
1566 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1567 {
1568         struct nand_chip *chip = &host->chip;
1569         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1570         int i;
1571
1572         nandc_read_buffer_sync(nandc, true);
1573
1574         for (i = 0; i < cw_cnt; i++) {
1575                 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1576
1577                 if (flash & (FS_OP_ERR | FS_MPU_ERR))
1578                         return -EIO;
1579         }
1580
1581         return 0;
1582 }
1583
1584 /* performs raw read for one codeword */
1585 static int
1586 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1587                        u8 *data_buf, u8 *oob_buf, int page, int cw)
1588 {
1589         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1590         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1591         struct nand_ecc_ctrl *ecc = &chip->ecc;
1592         int data_size1, data_size2, oob_size1, oob_size2;
1593         int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1594
1595         nand_read_page_op(chip, page, 0, NULL, 0);
1596         host->use_ecc = false;
1597
1598         clear_bam_transaction(nandc);
1599         set_address(host, host->cw_size * cw, page);
1600         update_rw_regs(host, 1, true);
1601         config_nand_page_read(nandc);
1602
1603         data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1604         oob_size1 = host->bbm_size;
1605
1606         if (cw == (ecc->steps - 1)) {
1607                 data_size2 = ecc->size - data_size1 -
1608                              ((ecc->steps - 1) * 4);
1609                 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1610                             host->spare_bytes;
1611         } else {
1612                 data_size2 = host->cw_data - data_size1;
1613                 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1614         }
1615
1616         if (nandc->props->is_bam) {
1617                 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1618                 read_loc += data_size1;
1619
1620                 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1621                 read_loc += oob_size1;
1622
1623                 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1624                 read_loc += data_size2;
1625
1626                 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1627         }
1628
1629         config_nand_cw_read(nandc, false);
1630
1631         read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1632         reg_off += data_size1;
1633
1634         read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1635         reg_off += oob_size1;
1636
1637         read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1638         reg_off += data_size2;
1639
1640         read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1641
1642         ret = submit_descs(nandc);
1643         free_descs(nandc);
1644         if (ret) {
1645                 dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1646                 return ret;
1647         }
1648
1649         return check_flash_errors(host, 1);
1650 }
1651
1652 /*
1653  * Bitflips can happen in erased codewords also so this function counts the
1654  * number of 0 in each CW for which ECC engine returns the uncorrectable
1655  * error. The page will be assumed as erased if this count is less than or
1656  * equal to the ecc->strength for each CW.
1657  *
1658  * 1. Both DATA and OOB need to be checked for number of 0. The
1659  *    top-level API can be called with only data buf or OOB buf so use
1660  *    chip->data_buf if data buf is null and chip->oob_poi if oob buf
1661  *    is null for copying the raw bytes.
1662  * 2. Perform raw read for all the CW which has uncorrectable errors.
1663  * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1664  *    The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1665  *    the number of bitflips in this area.
1666  */
1667 static int
1668 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1669                       u8 *oob_buf, unsigned long uncorrectable_cws,
1670                       int page, unsigned int max_bitflips)
1671 {
1672         struct nand_chip *chip = &host->chip;
1673         struct mtd_info *mtd = nand_to_mtd(chip);
1674         struct nand_ecc_ctrl *ecc = &chip->ecc;
1675         u8 *cw_data_buf, *cw_oob_buf;
1676         int cw, data_size, oob_size, ret = 0;
1677
1678         if (!data_buf)
1679                 data_buf = nand_get_data_buf(chip);
1680
1681         if (!oob_buf) {
1682                 nand_get_data_buf(chip);
1683                 oob_buf = chip->oob_poi;
1684         }
1685
1686         for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1687                 if (cw == (ecc->steps - 1)) {
1688                         data_size = ecc->size - ((ecc->steps - 1) * 4);
1689                         oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1690                 } else {
1691                         data_size = host->cw_data;
1692                         oob_size = host->ecc_bytes_hw;
1693                 }
1694
1695                 /* determine starting buffer address for current CW */
1696                 cw_data_buf = data_buf + (cw * host->cw_data);
1697                 cw_oob_buf = oob_buf + (cw * ecc->bytes);
1698
1699                 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1700                                              cw_oob_buf, page, cw);
1701                 if (ret)
1702                         return ret;
1703
1704                 /*
1705                  * make sure it isn't an erased page reported
1706                  * as not-erased by HW because of a few bitflips
1707                  */
1708                 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1709                                                   cw_oob_buf + host->bbm_size,
1710                                                   oob_size, NULL,
1711                                                   0, ecc->strength);
1712                 if (ret < 0) {
1713                         mtd->ecc_stats.failed++;
1714                 } else {
1715                         mtd->ecc_stats.corrected += ret;
1716                         max_bitflips = max_t(unsigned int, max_bitflips, ret);
1717                 }
1718         }
1719
1720         return max_bitflips;
1721 }
1722
1723 /*
1724  * reads back status registers set by the controller to notify page read
1725  * errors. this is equivalent to what 'ecc->correct()' would do.
1726  */
1727 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1728                              u8 *oob_buf, int page)
1729 {
1730         struct nand_chip *chip = &host->chip;
1731         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1732         struct mtd_info *mtd = nand_to_mtd(chip);
1733         struct nand_ecc_ctrl *ecc = &chip->ecc;
1734         unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1735         struct read_stats *buf;
1736         bool flash_op_err = false, erased;
1737         int i;
1738         u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1739
1740         buf = (struct read_stats *)nandc->reg_read_buf;
1741         nandc_read_buffer_sync(nandc, true);
1742
1743         for (i = 0; i < ecc->steps; i++, buf++) {
1744                 u32 flash, buffer, erased_cw;
1745                 int data_len, oob_len;
1746
1747                 if (i == (ecc->steps - 1)) {
1748                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1749                         oob_len = ecc->steps << 2;
1750                 } else {
1751                         data_len = host->cw_data;
1752                         oob_len = 0;
1753                 }
1754
1755                 flash = le32_to_cpu(buf->flash);
1756                 buffer = le32_to_cpu(buf->buffer);
1757                 erased_cw = le32_to_cpu(buf->erased_cw);
1758
1759                 /*
1760                  * Check ECC failure for each codeword. ECC failure can
1761                  * happen in either of the following conditions
1762                  * 1. If number of bitflips are greater than ECC engine
1763                  *    capability.
1764                  * 2. If this codeword contains all 0xff for which erased
1765                  *    codeword detection check will be done.
1766                  */
1767                 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1768                         /*
1769                          * For BCH ECC, ignore erased codeword errors, if
1770                          * ERASED_CW bits are set.
1771                          */
1772                         if (host->bch_enabled) {
1773                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1774                                          true : false;
1775                         /*
1776                          * For RS ECC, HW reports the erased CW by placing
1777                          * special characters at certain offsets in the buffer.
1778                          * These special characters will be valid only if
1779                          * complete page is read i.e. data_buf is not NULL.
1780                          */
1781                         } else if (data_buf) {
1782                                 erased = erased_chunk_check_and_fixup(data_buf,
1783                                                                       data_len);
1784                         } else {
1785                                 erased = false;
1786                         }
1787
1788                         if (!erased)
1789                                 uncorrectable_cws |= BIT(i);
1790                 /*
1791                  * Check if MPU or any other operational error (timeout,
1792                  * device failure, etc.) happened for this codeword and
1793                  * make flash_op_err true. If flash_op_err is set, then
1794                  * EIO will be returned for page read.
1795                  */
1796                 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1797                         flash_op_err = true;
1798                 /*
1799                  * No ECC or operational errors happened. Check the number of
1800                  * bits corrected and update the ecc_stats.corrected.
1801                  */
1802                 } else {
1803                         unsigned int stat;
1804
1805                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1806                         mtd->ecc_stats.corrected += stat;
1807                         max_bitflips = max(max_bitflips, stat);
1808                 }
1809
1810                 if (data_buf)
1811                         data_buf += data_len;
1812                 if (oob_buf)
1813                         oob_buf += oob_len + ecc->bytes;
1814         }
1815
1816         if (flash_op_err)
1817                 return -EIO;
1818
1819         if (!uncorrectable_cws)
1820                 return max_bitflips;
1821
1822         return check_for_erased_page(host, data_buf_start, oob_buf_start,
1823                                      uncorrectable_cws, page,
1824                                      max_bitflips);
1825 }
1826
1827 /*
1828  * helper to perform the actual page read operation, used by ecc->read_page(),
1829  * ecc->read_oob()
1830  */
1831 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1832                          u8 *oob_buf, int page)
1833 {
1834         struct nand_chip *chip = &host->chip;
1835         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1836         struct nand_ecc_ctrl *ecc = &chip->ecc;
1837         u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1838         int i, ret;
1839
1840         config_nand_page_read(nandc);
1841
1842         /* queue cmd descs for each codeword */
1843         for (i = 0; i < ecc->steps; i++) {
1844                 int data_size, oob_size;
1845
1846                 if (i == (ecc->steps - 1)) {
1847                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1848                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1849                                    host->spare_bytes;
1850                 } else {
1851                         data_size = host->cw_data;
1852                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1853                 }
1854
1855                 if (nandc->props->is_bam) {
1856                         if (data_buf && oob_buf) {
1857                                 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1858                                 nandc_set_read_loc(nandc, 1, data_size,
1859                                                    oob_size, 1);
1860                         } else if (data_buf) {
1861                                 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1862                         } else {
1863                                 nandc_set_read_loc(nandc, 0, data_size,
1864                                                    oob_size, 1);
1865                         }
1866                 }
1867
1868                 config_nand_cw_read(nandc, true);
1869
1870                 if (data_buf)
1871                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1872                                       data_size, 0);
1873
1874                 /*
1875                  * when ecc is enabled, the controller doesn't read the real
1876                  * or dummy bad block markers in each chunk. To maintain a
1877                  * consistent layout across RAW and ECC reads, we just
1878                  * leave the real/dummy BBM offsets empty (i.e, filled with
1879                  * 0xffs)
1880                  */
1881                 if (oob_buf) {
1882                         int j;
1883
1884                         for (j = 0; j < host->bbm_size; j++)
1885                                 *oob_buf++ = 0xff;
1886
1887                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1888                                       oob_buf, oob_size, 0);
1889                 }
1890
1891                 if (data_buf)
1892                         data_buf += data_size;
1893                 if (oob_buf)
1894                         oob_buf += oob_size;
1895         }
1896
1897         ret = submit_descs(nandc);
1898         free_descs(nandc);
1899
1900         if (ret) {
1901                 dev_err(nandc->dev, "failure to read page/oob\n");
1902                 return ret;
1903         }
1904
1905         return parse_read_errors(host, data_buf_start, oob_buf_start, page);
1906 }
1907
1908 /*
1909  * a helper that copies the last step/codeword of a page (containing free oob)
1910  * into our local buffer
1911  */
1912 static int copy_last_cw(struct qcom_nand_host *host, int page)
1913 {
1914         struct nand_chip *chip = &host->chip;
1915         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1916         struct nand_ecc_ctrl *ecc = &chip->ecc;
1917         int size;
1918         int ret;
1919
1920         clear_read_regs(nandc);
1921
1922         size = host->use_ecc ? host->cw_data : host->cw_size;
1923
1924         /* prepare a clean read buffer */
1925         memset(nandc->data_buffer, 0xff, size);
1926
1927         set_address(host, host->cw_size * (ecc->steps - 1), page);
1928         update_rw_regs(host, 1, true);
1929
1930         config_nand_single_cw_page_read(nandc, host->use_ecc);
1931
1932         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1933
1934         ret = submit_descs(nandc);
1935         if (ret)
1936                 dev_err(nandc->dev, "failed to copy last codeword\n");
1937
1938         free_descs(nandc);
1939
1940         return ret;
1941 }
1942
1943 /* implements ecc->read_page() */
1944 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf,
1945                                 int oob_required, int page)
1946 {
1947         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1948         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1949         u8 *data_buf, *oob_buf = NULL;
1950
1951         nand_read_page_op(chip, page, 0, NULL, 0);
1952         data_buf = buf;
1953         oob_buf = oob_required ? chip->oob_poi : NULL;
1954
1955         clear_bam_transaction(nandc);
1956
1957         return read_page_ecc(host, data_buf, oob_buf, page);
1958 }
1959
1960 /* implements ecc->read_page_raw() */
1961 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
1962                                     int oob_required, int page)
1963 {
1964         struct mtd_info *mtd = nand_to_mtd(chip);
1965         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1966         struct nand_ecc_ctrl *ecc = &chip->ecc;
1967         int cw, ret;
1968         u8 *data_buf = buf, *oob_buf = chip->oob_poi;
1969
1970         for (cw = 0; cw < ecc->steps; cw++) {
1971                 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
1972                                              page, cw);
1973                 if (ret)
1974                         return ret;
1975
1976                 data_buf += host->cw_data;
1977                 oob_buf += ecc->bytes;
1978         }
1979
1980         return 0;
1981 }
1982
1983 /* implements ecc->read_oob() */
1984 static int qcom_nandc_read_oob(struct nand_chip *chip, int page)
1985 {
1986         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1987         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1988         struct nand_ecc_ctrl *ecc = &chip->ecc;
1989
1990         clear_read_regs(nandc);
1991         clear_bam_transaction(nandc);
1992
1993         host->use_ecc = true;
1994         set_address(host, 0, page);
1995         update_rw_regs(host, ecc->steps, true);
1996
1997         return read_page_ecc(host, NULL, chip->oob_poi, page);
1998 }
1999
2000 /* implements ecc->write_page() */
2001 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf,
2002                                  int oob_required, int page)
2003 {
2004         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2005         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2006         struct nand_ecc_ctrl *ecc = &chip->ecc;
2007         u8 *data_buf, *oob_buf;
2008         int i, ret;
2009
2010         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2011
2012         clear_read_regs(nandc);
2013         clear_bam_transaction(nandc);
2014
2015         data_buf = (u8 *)buf;
2016         oob_buf = chip->oob_poi;
2017
2018         host->use_ecc = true;
2019         update_rw_regs(host, ecc->steps, false);
2020         config_nand_page_write(nandc);
2021
2022         for (i = 0; i < ecc->steps; i++) {
2023                 int data_size, oob_size;
2024
2025                 if (i == (ecc->steps - 1)) {
2026                         data_size = ecc->size - ((ecc->steps - 1) << 2);
2027                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2028                                    host->spare_bytes;
2029                 } else {
2030                         data_size = host->cw_data;
2031                         oob_size = ecc->bytes;
2032                 }
2033
2034
2035                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2036                                i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2037
2038                 /*
2039                  * when ECC is enabled, we don't really need to write anything
2040                  * to oob for the first n - 1 codewords since these oob regions
2041                  * just contain ECC bytes that's written by the controller
2042                  * itself. For the last codeword, we skip the bbm positions and
2043                  * write to the free oob area.
2044                  */
2045                 if (i == (ecc->steps - 1)) {
2046                         oob_buf += host->bbm_size;
2047
2048                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2049                                        oob_buf, oob_size, 0);
2050                 }
2051
2052                 config_nand_cw_write(nandc);
2053
2054                 data_buf += data_size;
2055                 oob_buf += oob_size;
2056         }
2057
2058         ret = submit_descs(nandc);
2059         if (ret)
2060                 dev_err(nandc->dev, "failure to write page\n");
2061
2062         free_descs(nandc);
2063
2064         if (!ret)
2065                 ret = nand_prog_page_end_op(chip);
2066
2067         return ret;
2068 }
2069
2070 /* implements ecc->write_page_raw() */
2071 static int qcom_nandc_write_page_raw(struct nand_chip *chip,
2072                                      const uint8_t *buf, int oob_required,
2073                                      int page)
2074 {
2075         struct mtd_info *mtd = nand_to_mtd(chip);
2076         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2077         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2078         struct nand_ecc_ctrl *ecc = &chip->ecc;
2079         u8 *data_buf, *oob_buf;
2080         int i, ret;
2081
2082         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2083         clear_read_regs(nandc);
2084         clear_bam_transaction(nandc);
2085
2086         data_buf = (u8 *)buf;
2087         oob_buf = chip->oob_poi;
2088
2089         host->use_ecc = false;
2090         update_rw_regs(host, ecc->steps, false);
2091         config_nand_page_write(nandc);
2092
2093         for (i = 0; i < ecc->steps; i++) {
2094                 int data_size1, data_size2, oob_size1, oob_size2;
2095                 int reg_off = FLASH_BUF_ACC;
2096
2097                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2098                 oob_size1 = host->bbm_size;
2099
2100                 if (i == (ecc->steps - 1)) {
2101                         data_size2 = ecc->size - data_size1 -
2102                                      ((ecc->steps - 1) << 2);
2103                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2104                                     host->spare_bytes;
2105                 } else {
2106                         data_size2 = host->cw_data - data_size1;
2107                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2108                 }
2109
2110                 write_data_dma(nandc, reg_off, data_buf, data_size1,
2111                                NAND_BAM_NO_EOT);
2112                 reg_off += data_size1;
2113                 data_buf += data_size1;
2114
2115                 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2116                                NAND_BAM_NO_EOT);
2117                 reg_off += oob_size1;
2118                 oob_buf += oob_size1;
2119
2120                 write_data_dma(nandc, reg_off, data_buf, data_size2,
2121                                NAND_BAM_NO_EOT);
2122                 reg_off += data_size2;
2123                 data_buf += data_size2;
2124
2125                 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2126                 oob_buf += oob_size2;
2127
2128                 config_nand_cw_write(nandc);
2129         }
2130
2131         ret = submit_descs(nandc);
2132         if (ret)
2133                 dev_err(nandc->dev, "failure to write raw page\n");
2134
2135         free_descs(nandc);
2136
2137         if (!ret)
2138                 ret = nand_prog_page_end_op(chip);
2139
2140         return ret;
2141 }
2142
2143 /*
2144  * implements ecc->write_oob()
2145  *
2146  * the NAND controller cannot write only data or only OOB within a codeword
2147  * since ECC is calculated for the combined codeword. So update the OOB from
2148  * chip->oob_poi, and pad the data area with OxFF before writing.
2149  */
2150 static int qcom_nandc_write_oob(struct nand_chip *chip, int page)
2151 {
2152         struct mtd_info *mtd = nand_to_mtd(chip);
2153         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2154         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2155         struct nand_ecc_ctrl *ecc = &chip->ecc;
2156         u8 *oob = chip->oob_poi;
2157         int data_size, oob_size;
2158         int ret;
2159
2160         host->use_ecc = true;
2161         clear_bam_transaction(nandc);
2162
2163         /* calculate the data and oob size for the last codeword/step */
2164         data_size = ecc->size - ((ecc->steps - 1) << 2);
2165         oob_size = mtd->oobavail;
2166
2167         memset(nandc->data_buffer, 0xff, host->cw_data);
2168         /* override new oob content to last codeword */
2169         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2170                                     0, mtd->oobavail);
2171
2172         set_address(host, host->cw_size * (ecc->steps - 1), page);
2173         update_rw_regs(host, 1, false);
2174
2175         config_nand_page_write(nandc);
2176         write_data_dma(nandc, FLASH_BUF_ACC,
2177                        nandc->data_buffer, data_size + oob_size, 0);
2178         config_nand_cw_write(nandc);
2179
2180         ret = submit_descs(nandc);
2181
2182         free_descs(nandc);
2183
2184         if (ret) {
2185                 dev_err(nandc->dev, "failure to write oob\n");
2186                 return -EIO;
2187         }
2188
2189         return nand_prog_page_end_op(chip);
2190 }
2191
2192 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs)
2193 {
2194         struct mtd_info *mtd = nand_to_mtd(chip);
2195         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2196         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2197         struct nand_ecc_ctrl *ecc = &chip->ecc;
2198         int page, ret, bbpos, bad = 0;
2199
2200         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2201
2202         /*
2203          * configure registers for a raw sub page read, the address is set to
2204          * the beginning of the last codeword, we don't care about reading ecc
2205          * portion of oob. we just want the first few bytes from this codeword
2206          * that contains the BBM
2207          */
2208         host->use_ecc = false;
2209
2210         clear_bam_transaction(nandc);
2211         ret = copy_last_cw(host, page);
2212         if (ret)
2213                 goto err;
2214
2215         if (check_flash_errors(host, 1)) {
2216                 dev_warn(nandc->dev, "error when trying to read BBM\n");
2217                 goto err;
2218         }
2219
2220         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2221
2222         bad = nandc->data_buffer[bbpos] != 0xff;
2223
2224         if (chip->options & NAND_BUSWIDTH_16)
2225                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2226 err:
2227         return bad;
2228 }
2229
2230 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs)
2231 {
2232         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2233         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2234         struct nand_ecc_ctrl *ecc = &chip->ecc;
2235         int page, ret;
2236
2237         clear_read_regs(nandc);
2238         clear_bam_transaction(nandc);
2239
2240         /*
2241          * to mark the BBM as bad, we flash the entire last codeword with 0s.
2242          * we don't care about the rest of the content in the codeword since
2243          * we aren't going to use this block again
2244          */
2245         memset(nandc->data_buffer, 0x00, host->cw_size);
2246
2247         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2248
2249         /* prepare write */
2250         host->use_ecc = false;
2251         set_address(host, host->cw_size * (ecc->steps - 1), page);
2252         update_rw_regs(host, 1, false);
2253
2254         config_nand_page_write(nandc);
2255         write_data_dma(nandc, FLASH_BUF_ACC,
2256                        nandc->data_buffer, host->cw_size, 0);
2257         config_nand_cw_write(nandc);
2258
2259         ret = submit_descs(nandc);
2260
2261         free_descs(nandc);
2262
2263         if (ret) {
2264                 dev_err(nandc->dev, "failure to update BBM\n");
2265                 return -EIO;
2266         }
2267
2268         return nand_prog_page_end_op(chip);
2269 }
2270
2271 /*
2272  * the three functions below implement chip->legacy.read_byte(),
2273  * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these
2274  * aren't used for reading/writing page data, they are used for smaller data
2275  * like reading id, status etc
2276  */
2277 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip)
2278 {
2279         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2280         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2281         u8 *buf = nandc->data_buffer;
2282         u8 ret = 0x0;
2283
2284         if (host->last_command == NAND_CMD_STATUS) {
2285                 ret = host->status;
2286
2287                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2288
2289                 return ret;
2290         }
2291
2292         if (nandc->buf_start < nandc->buf_count)
2293                 ret = buf[nandc->buf_start++];
2294
2295         return ret;
2296 }
2297
2298 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len)
2299 {
2300         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2301         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2302
2303         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2304         nandc->buf_start += real_len;
2305 }
2306
2307 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf,
2308                                  int len)
2309 {
2310         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2311         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2312
2313         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2314
2315         nandc->buf_start += real_len;
2316 }
2317
2318 /* we support only one external chip for now */
2319 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr)
2320 {
2321         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2322
2323         if (chipnr <= 0)
2324                 return;
2325
2326         dev_warn(nandc->dev, "invalid chip select\n");
2327 }
2328
2329 /*
2330  * NAND controller page layout info
2331  *
2332  * Layout with ECC enabled:
2333  *
2334  * |----------------------|  |---------------------------------|
2335  * |           xx.......yy|  |             *********xx.......yy|
2336  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
2337  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
2338  * |           xx.......yy|  |             *********xx.......yy|
2339  * |----------------------|  |---------------------------------|
2340  *     codeword 1,2..n-1                  codeword n
2341  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
2342  *
2343  * n = Number of codewords in the page
2344  * . = ECC bytes
2345  * * = Spare/free bytes
2346  * x = Unused byte(s)
2347  * y = Reserved byte(s)
2348  *
2349  * 2K page: n = 4, spare = 16 bytes
2350  * 4K page: n = 8, spare = 32 bytes
2351  * 8K page: n = 16, spare = 64 bytes
2352  *
2353  * the qcom nand controller operates at a sub page/codeword level. each
2354  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2355  * the number of ECC bytes vary based on the ECC strength and the bus width.
2356  *
2357  * the first n - 1 codewords contains 516 bytes of user data, the remaining
2358  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2359  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2360  *
2361  * When we access a page with ECC enabled, the reserved bytes(s) are not
2362  * accessible at all. When reading, we fill up these unreadable positions
2363  * with 0xffs. When writing, the controller skips writing the inaccessible
2364  * bytes.
2365  *
2366  * Layout with ECC disabled:
2367  *
2368  * |------------------------------|  |---------------------------------------|
2369  * |         yy          xx.......|  |         bb          *********xx.......|
2370  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
2371  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
2372  * |         yy          xx.......|  |         bb          *********xx.......|
2373  * |------------------------------|  |---------------------------------------|
2374  *         codeword 1,2..n-1                        codeword n
2375  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
2376  *
2377  * n = Number of codewords in the page
2378  * . = ECC bytes
2379  * * = Spare/free bytes
2380  * x = Unused byte(s)
2381  * y = Dummy Bad Bock byte(s)
2382  * b = Real Bad Block byte(s)
2383  * size1/size2 = function of codeword size and 'n'
2384  *
2385  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2386  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2387  * Block Markers. In the last codeword, this position contains the real BBM
2388  *
2389  * In order to have a consistent layout between RAW and ECC modes, we assume
2390  * the following OOB layout arrangement:
2391  *
2392  * |-----------|  |--------------------|
2393  * |yyxx.......|  |bb*********xx.......|
2394  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
2395  * |yyxx.......|  |bb*********xx.......|
2396  * |yyxx.......|  |bb*********xx.......|
2397  * |-----------|  |--------------------|
2398  *  first n - 1       nth OOB region
2399  *  OOB regions
2400  *
2401  * n = Number of codewords in the page
2402  * . = ECC bytes
2403  * * = FREE OOB bytes
2404  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2405  * x = Unused byte(s)
2406  * b = Real bad block byte(s) (inaccessible when ECC enabled)
2407  *
2408  * This layout is read as is when ECC is disabled. When ECC is enabled, the
2409  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2410  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2411  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2412  * the sum of the three).
2413  */
2414 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2415                                    struct mtd_oob_region *oobregion)
2416 {
2417         struct nand_chip *chip = mtd_to_nand(mtd);
2418         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2419         struct nand_ecc_ctrl *ecc = &chip->ecc;
2420
2421         if (section > 1)
2422                 return -ERANGE;
2423
2424         if (!section) {
2425                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2426                                     host->bbm_size;
2427                 oobregion->offset = 0;
2428         } else {
2429                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2430                 oobregion->offset = mtd->oobsize - oobregion->length;
2431         }
2432
2433         return 0;
2434 }
2435
2436 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2437                                      struct mtd_oob_region *oobregion)
2438 {
2439         struct nand_chip *chip = mtd_to_nand(mtd);
2440         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2441         struct nand_ecc_ctrl *ecc = &chip->ecc;
2442
2443         if (section)
2444                 return -ERANGE;
2445
2446         oobregion->length = ecc->steps * 4;
2447         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2448
2449         return 0;
2450 }
2451
2452 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2453         .ecc = qcom_nand_ooblayout_ecc,
2454         .free = qcom_nand_ooblayout_free,
2455 };
2456
2457 static int
2458 qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2459 {
2460         return strength == 4 ? 12 : 16;
2461 }
2462 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2463                      NANDC_STEP_SIZE, 4, 8);
2464
2465 static int qcom_nand_attach_chip(struct nand_chip *chip)
2466 {
2467         struct mtd_info *mtd = nand_to_mtd(chip);
2468         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2469         struct nand_ecc_ctrl *ecc = &chip->ecc;
2470         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2471         int cwperpage, bad_block_byte, ret;
2472         bool wide_bus;
2473         int ecc_mode = 1;
2474
2475         /* controller only supports 512 bytes data steps */
2476         ecc->size = NANDC_STEP_SIZE;
2477         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2478         cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2479
2480         /*
2481          * Each CW has 4 available OOB bytes which will be protected with ECC
2482          * so remaining bytes can be used for ECC.
2483          */
2484         ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2485                                    mtd->oobsize - (cwperpage * 4));
2486         if (ret) {
2487                 dev_err(nandc->dev, "No valid ECC settings possible\n");
2488                 return ret;
2489         }
2490
2491         if (ecc->strength >= 8) {
2492                 /* 8 bit ECC defaults to BCH ECC on all platforms */
2493                 host->bch_enabled = true;
2494                 ecc_mode = 1;
2495
2496                 if (wide_bus) {
2497                         host->ecc_bytes_hw = 14;
2498                         host->spare_bytes = 0;
2499                         host->bbm_size = 2;
2500                 } else {
2501                         host->ecc_bytes_hw = 13;
2502                         host->spare_bytes = 2;
2503                         host->bbm_size = 1;
2504                 }
2505         } else {
2506                 /*
2507                  * if the controller supports BCH for 4 bit ECC, the controller
2508                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2509                  * always 10 bytes
2510                  */
2511                 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2512                         /* BCH */
2513                         host->bch_enabled = true;
2514                         ecc_mode = 0;
2515
2516                         if (wide_bus) {
2517                                 host->ecc_bytes_hw = 8;
2518                                 host->spare_bytes = 2;
2519                                 host->bbm_size = 2;
2520                         } else {
2521                                 host->ecc_bytes_hw = 7;
2522                                 host->spare_bytes = 4;
2523                                 host->bbm_size = 1;
2524                         }
2525                 } else {
2526                         /* RS */
2527                         host->ecc_bytes_hw = 10;
2528
2529                         if (wide_bus) {
2530                                 host->spare_bytes = 0;
2531                                 host->bbm_size = 2;
2532                         } else {
2533                                 host->spare_bytes = 1;
2534                                 host->bbm_size = 1;
2535                         }
2536                 }
2537         }
2538
2539         /*
2540          * we consider ecc->bytes as the sum of all the non-data content in a
2541          * step. It gives us a clean representation of the oob area (even if
2542          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2543          * ECC and 12 bytes for 4 bit ECC
2544          */
2545         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2546
2547         ecc->read_page          = qcom_nandc_read_page;
2548         ecc->read_page_raw      = qcom_nandc_read_page_raw;
2549         ecc->read_oob           = qcom_nandc_read_oob;
2550         ecc->write_page         = qcom_nandc_write_page;
2551         ecc->write_page_raw     = qcom_nandc_write_page_raw;
2552         ecc->write_oob          = qcom_nandc_write_oob;
2553
2554         ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2555
2556         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2557
2558         nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2559                                      cwperpage);
2560
2561         /*
2562          * DATA_UD_BYTES varies based on whether the read/write command protects
2563          * spare data with ECC too. We protect spare data by default, so we set
2564          * it to main + spare data, which are 512 and 4 bytes respectively.
2565          */
2566         host->cw_data = 516;
2567
2568         /*
2569          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2570          * for 8 bit ECC
2571          */
2572         host->cw_size = host->cw_data + ecc->bytes;
2573         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2574
2575         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2576                                 | host->cw_data << UD_SIZE_BYTES
2577                                 | 0 << DISABLE_STATUS_AFTER_WRITE
2578                                 | 5 << NUM_ADDR_CYCLES
2579                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2580                                 | 0 << STATUS_BFR_READ
2581                                 | 1 << SET_RD_MODE_AFTER_STATUS
2582                                 | host->spare_bytes << SPARE_SIZE_BYTES;
2583
2584         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2585                                 | 0 <<  CS_ACTIVE_BSY
2586                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2587                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
2588                                 | 2 << WR_RD_BSY_GAP
2589                                 | wide_bus << WIDE_FLASH
2590                                 | host->bch_enabled << ENABLE_BCH_ECC;
2591
2592         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2593                                 | host->cw_size << UD_SIZE_BYTES
2594                                 | 5 << NUM_ADDR_CYCLES
2595                                 | 0 << SPARE_SIZE_BYTES;
2596
2597         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2598                                 | 0 << CS_ACTIVE_BSY
2599                                 | 17 << BAD_BLOCK_BYTE_NUM
2600                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
2601                                 | 2 << WR_RD_BSY_GAP
2602                                 | wide_bus << WIDE_FLASH
2603                                 | 1 << DEV0_CFG1_ECC_DISABLE;
2604
2605         host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2606                                 | 0 << ECC_SW_RESET
2607                                 | host->cw_data << ECC_NUM_DATA_BYTES
2608                                 | 1 << ECC_FORCE_CLK_OPEN
2609                                 | ecc_mode << ECC_MODE
2610                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2611
2612         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2613
2614         host->clrflashstatus = FS_READY_BSY_N;
2615         host->clrreadstatus = 0xc0;
2616         nandc->regs->erased_cw_detect_cfg_clr =
2617                 cpu_to_le32(CLR_ERASED_PAGE_DET);
2618         nandc->regs->erased_cw_detect_cfg_set =
2619                 cpu_to_le32(SET_ERASED_PAGE_DET);
2620
2621         dev_dbg(nandc->dev,
2622                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2623                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2624                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2625                 cwperpage);
2626
2627         return 0;
2628 }
2629
2630 static const struct nand_controller_ops qcom_nandc_ops = {
2631         .attach_chip = qcom_nand_attach_chip,
2632 };
2633
2634 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2635 {
2636         if (nandc->props->is_bam) {
2637                 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2638                         dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2639                                          MAX_REG_RD *
2640                                          sizeof(*nandc->reg_read_buf),
2641                                          DMA_FROM_DEVICE);
2642
2643                 if (nandc->tx_chan)
2644                         dma_release_channel(nandc->tx_chan);
2645
2646                 if (nandc->rx_chan)
2647                         dma_release_channel(nandc->rx_chan);
2648
2649                 if (nandc->cmd_chan)
2650                         dma_release_channel(nandc->cmd_chan);
2651         } else {
2652                 if (nandc->chan)
2653                         dma_release_channel(nandc->chan);
2654         }
2655 }
2656
2657 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2658 {
2659         int ret;
2660
2661         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2662         if (ret) {
2663                 dev_err(nandc->dev, "failed to set DMA mask\n");
2664                 return ret;
2665         }
2666
2667         /*
2668          * we use the internal buffer for reading ONFI params, reading small
2669          * data like ID and status, and preforming read-copy-write operations
2670          * when writing to a codeword partially. 532 is the maximum possible
2671          * size of a codeword for our nand controller
2672          */
2673         nandc->buf_size = 532;
2674
2675         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2676                                         GFP_KERNEL);
2677         if (!nandc->data_buffer)
2678                 return -ENOMEM;
2679
2680         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2681                                         GFP_KERNEL);
2682         if (!nandc->regs)
2683                 return -ENOMEM;
2684
2685         nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2686                                 MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2687                                 GFP_KERNEL);
2688         if (!nandc->reg_read_buf)
2689                 return -ENOMEM;
2690
2691         if (nandc->props->is_bam) {
2692                 nandc->reg_read_dma =
2693                         dma_map_single(nandc->dev, nandc->reg_read_buf,
2694                                        MAX_REG_RD *
2695                                        sizeof(*nandc->reg_read_buf),
2696                                        DMA_FROM_DEVICE);
2697                 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2698                         dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2699                         return -EIO;
2700                 }
2701
2702                 nandc->tx_chan = dma_request_chan(nandc->dev, "tx");
2703                 if (IS_ERR(nandc->tx_chan)) {
2704                         ret = PTR_ERR(nandc->tx_chan);
2705                         nandc->tx_chan = NULL;
2706                         dev_err_probe(nandc->dev, ret,
2707                                       "tx DMA channel request failed\n");
2708                         goto unalloc;
2709                 }
2710
2711                 nandc->rx_chan = dma_request_chan(nandc->dev, "rx");
2712                 if (IS_ERR(nandc->rx_chan)) {
2713                         ret = PTR_ERR(nandc->rx_chan);
2714                         nandc->rx_chan = NULL;
2715                         dev_err_probe(nandc->dev, ret,
2716                                       "rx DMA channel request failed\n");
2717                         goto unalloc;
2718                 }
2719
2720                 nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd");
2721                 if (IS_ERR(nandc->cmd_chan)) {
2722                         ret = PTR_ERR(nandc->cmd_chan);
2723                         nandc->cmd_chan = NULL;
2724                         dev_err_probe(nandc->dev, ret,
2725                                       "cmd DMA channel request failed\n");
2726                         goto unalloc;
2727                 }
2728
2729                 /*
2730                  * Initially allocate BAM transaction to read ONFI param page.
2731                  * After detecting all the devices, this BAM transaction will
2732                  * be freed and the next BAM tranasction will be allocated with
2733                  * maximum codeword size
2734                  */
2735                 nandc->max_cwperpage = 1;
2736                 nandc->bam_txn = alloc_bam_transaction(nandc);
2737                 if (!nandc->bam_txn) {
2738                         dev_err(nandc->dev,
2739                                 "failed to allocate bam transaction\n");
2740                         ret = -ENOMEM;
2741                         goto unalloc;
2742                 }
2743         } else {
2744                 nandc->chan = dma_request_chan(nandc->dev, "rxtx");
2745                 if (IS_ERR(nandc->chan)) {
2746                         ret = PTR_ERR(nandc->chan);
2747                         nandc->chan = NULL;
2748                         dev_err_probe(nandc->dev, ret,
2749                                       "rxtx DMA channel request failed\n");
2750                         return ret;
2751                 }
2752         }
2753
2754         INIT_LIST_HEAD(&nandc->desc_list);
2755         INIT_LIST_HEAD(&nandc->host_list);
2756
2757         nand_controller_init(&nandc->controller);
2758         nandc->controller.ops = &qcom_nandc_ops;
2759
2760         return 0;
2761 unalloc:
2762         qcom_nandc_unalloc(nandc);
2763         return ret;
2764 }
2765
2766 /* one time setup of a few nand controller registers */
2767 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2768 {
2769         u32 nand_ctrl;
2770
2771         /* kill onenand */
2772         if (!nandc->props->is_qpic)
2773                 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2774         nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2775                     NAND_DEV_CMD_VLD_VAL);
2776
2777         /* enable ADM or BAM DMA */
2778         if (nandc->props->is_bam) {
2779                 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2780
2781                 /*
2782                  *NAND_CTRL is an operational registers, and CPU
2783                  * access to operational registers are read only
2784                  * in BAM mode. So update the NAND_CTRL register
2785                  * only if it is not in BAM mode. In most cases BAM
2786                  * mode will be enabled in bootloader
2787                  */
2788                 if (!(nand_ctrl & BAM_MODE_EN))
2789                         nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2790         } else {
2791                 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2792         }
2793
2794         /* save the original values of these registers */
2795         nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2796         nandc->vld = NAND_DEV_CMD_VLD_VAL;
2797
2798         return 0;
2799 }
2800
2801 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2802                                             struct qcom_nand_host *host,
2803                                             struct device_node *dn)
2804 {
2805         struct nand_chip *chip = &host->chip;
2806         struct mtd_info *mtd = nand_to_mtd(chip);
2807         struct device *dev = nandc->dev;
2808         int ret;
2809
2810         ret = of_property_read_u32(dn, "reg", &host->cs);
2811         if (ret) {
2812                 dev_err(dev, "can't get chip-select\n");
2813                 return -ENXIO;
2814         }
2815
2816         nand_set_flash_node(chip, dn);
2817         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2818         if (!mtd->name)
2819                 return -ENOMEM;
2820
2821         mtd->owner = THIS_MODULE;
2822         mtd->dev.parent = dev;
2823
2824         chip->legacy.cmdfunc    = qcom_nandc_command;
2825         chip->legacy.select_chip        = qcom_nandc_select_chip;
2826         chip->legacy.read_byte  = qcom_nandc_read_byte;
2827         chip->legacy.read_buf   = qcom_nandc_read_buf;
2828         chip->legacy.write_buf  = qcom_nandc_write_buf;
2829         chip->legacy.set_features       = nand_get_set_features_notsupp;
2830         chip->legacy.get_features       = nand_get_set_features_notsupp;
2831
2832         /*
2833          * the bad block marker is readable only when we read the last codeword
2834          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2835          * helpers don't allow us to read BB from a nand chip with ECC
2836          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2837          * and block_markbad helpers until we permanently switch to using
2838          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2839          */
2840         chip->legacy.block_bad          = qcom_nandc_block_bad;
2841         chip->legacy.block_markbad      = qcom_nandc_block_markbad;
2842
2843         chip->controller = &nandc->controller;
2844         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA |
2845                          NAND_SKIP_BBTSCAN;
2846
2847         /* set up initial status value */
2848         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2849
2850         ret = nand_scan(chip, 1);
2851         if (ret)
2852                 return ret;
2853
2854         if (nandc->props->is_bam) {
2855                 free_bam_transaction(nandc);
2856                 nandc->bam_txn = alloc_bam_transaction(nandc);
2857                 if (!nandc->bam_txn) {
2858                         dev_err(nandc->dev,
2859                                 "failed to allocate bam transaction\n");
2860                         return -ENOMEM;
2861                 }
2862         }
2863
2864         ret = mtd_device_register(mtd, NULL, 0);
2865         if (ret)
2866                 nand_cleanup(chip);
2867
2868         return ret;
2869 }
2870
2871 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2872 {
2873         struct device *dev = nandc->dev;
2874         struct device_node *dn = dev->of_node, *child;
2875         struct qcom_nand_host *host;
2876         int ret = -ENODEV;
2877
2878         for_each_available_child_of_node(dn, child) {
2879                 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2880                 if (!host) {
2881                         of_node_put(child);
2882                         return -ENOMEM;
2883                 }
2884
2885                 ret = qcom_nand_host_init_and_register(nandc, host, child);
2886                 if (ret) {
2887                         devm_kfree(dev, host);
2888                         continue;
2889                 }
2890
2891                 list_add_tail(&host->node, &nandc->host_list);
2892         }
2893
2894         return ret;
2895 }
2896
2897 /* parse custom DT properties here */
2898 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2899 {
2900         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2901         struct device_node *np = nandc->dev->of_node;
2902         int ret;
2903
2904         if (!nandc->props->is_bam) {
2905                 ret = of_property_read_u32(np, "qcom,cmd-crci",
2906                                            &nandc->cmd_crci);
2907                 if (ret) {
2908                         dev_err(nandc->dev, "command CRCI unspecified\n");
2909                         return ret;
2910                 }
2911
2912                 ret = of_property_read_u32(np, "qcom,data-crci",
2913                                            &nandc->data_crci);
2914                 if (ret) {
2915                         dev_err(nandc->dev, "data CRCI unspecified\n");
2916                         return ret;
2917                 }
2918         }
2919
2920         return 0;
2921 }
2922
2923 static int qcom_nandc_probe(struct platform_device *pdev)
2924 {
2925         struct qcom_nand_controller *nandc;
2926         const void *dev_data;
2927         struct device *dev = &pdev->dev;
2928         struct resource *res;
2929         int ret;
2930
2931         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2932         if (!nandc)
2933                 return -ENOMEM;
2934
2935         platform_set_drvdata(pdev, nandc);
2936         nandc->dev = dev;
2937
2938         dev_data = of_device_get_match_data(dev);
2939         if (!dev_data) {
2940                 dev_err(&pdev->dev, "failed to get device data\n");
2941                 return -ENODEV;
2942         }
2943
2944         nandc->props = dev_data;
2945
2946         nandc->core_clk = devm_clk_get(dev, "core");
2947         if (IS_ERR(nandc->core_clk))
2948                 return PTR_ERR(nandc->core_clk);
2949
2950         nandc->aon_clk = devm_clk_get(dev, "aon");
2951         if (IS_ERR(nandc->aon_clk))
2952                 return PTR_ERR(nandc->aon_clk);
2953
2954         ret = qcom_nandc_parse_dt(pdev);
2955         if (ret)
2956                 return ret;
2957
2958         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2959         nandc->base = devm_ioremap_resource(dev, res);
2960         if (IS_ERR(nandc->base))
2961                 return PTR_ERR(nandc->base);
2962
2963         nandc->base_phys = res->start;
2964         nandc->base_dma = dma_map_resource(dev, res->start,
2965                                            resource_size(res),
2966                                            DMA_BIDIRECTIONAL, 0);
2967         if (!nandc->base_dma)
2968                 return -ENXIO;
2969
2970         ret = clk_prepare_enable(nandc->core_clk);
2971         if (ret)
2972                 goto err_core_clk;
2973
2974         ret = clk_prepare_enable(nandc->aon_clk);
2975         if (ret)
2976                 goto err_aon_clk;
2977
2978         ret = qcom_nandc_alloc(nandc);
2979         if (ret)
2980                 goto err_nandc_alloc;
2981
2982         ret = qcom_nandc_setup(nandc);
2983         if (ret)
2984                 goto err_setup;
2985
2986         ret = qcom_probe_nand_devices(nandc);
2987         if (ret)
2988                 goto err_setup;
2989
2990         return 0;
2991
2992 err_setup:
2993         qcom_nandc_unalloc(nandc);
2994 err_nandc_alloc:
2995         clk_disable_unprepare(nandc->aon_clk);
2996 err_aon_clk:
2997         clk_disable_unprepare(nandc->core_clk);
2998 err_core_clk:
2999         dma_unmap_resource(dev, nandc->base_dma, resource_size(res),
3000                            DMA_BIDIRECTIONAL, 0);
3001         return ret;
3002 }
3003
3004 static int qcom_nandc_remove(struct platform_device *pdev)
3005 {
3006         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
3007         struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3008         struct qcom_nand_host *host;
3009         struct nand_chip *chip;
3010         int ret;
3011
3012         list_for_each_entry(host, &nandc->host_list, node) {
3013                 chip = &host->chip;
3014                 ret = mtd_device_unregister(nand_to_mtd(chip));
3015                 WARN_ON(ret);
3016                 nand_cleanup(chip);
3017         }
3018
3019         qcom_nandc_unalloc(nandc);
3020
3021         clk_disable_unprepare(nandc->aon_clk);
3022         clk_disable_unprepare(nandc->core_clk);
3023
3024         dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3025                            DMA_BIDIRECTIONAL, 0);
3026
3027         return 0;
3028 }
3029
3030 static const struct qcom_nandc_props ipq806x_nandc_props = {
3031         .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3032         .is_bam = false,
3033         .dev_cmd_reg_start = 0x0,
3034 };
3035
3036 static const struct qcom_nandc_props ipq4019_nandc_props = {
3037         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3038         .is_bam = true,
3039         .is_qpic = true,
3040         .dev_cmd_reg_start = 0x0,
3041 };
3042
3043 static const struct qcom_nandc_props ipq8074_nandc_props = {
3044         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3045         .is_bam = true,
3046         .is_qpic = true,
3047         .dev_cmd_reg_start = 0x7000,
3048 };
3049
3050 /*
3051  * data will hold a struct pointer containing more differences once we support
3052  * more controller variants
3053  */
3054 static const struct of_device_id qcom_nandc_of_match[] = {
3055         {
3056                 .compatible = "qcom,ipq806x-nand",
3057                 .data = &ipq806x_nandc_props,
3058         },
3059         {
3060                 .compatible = "qcom,ipq4019-nand",
3061                 .data = &ipq4019_nandc_props,
3062         },
3063         {
3064                 .compatible = "qcom,ipq8074-nand",
3065                 .data = &ipq8074_nandc_props,
3066         },
3067         {}
3068 };
3069 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3070
3071 static struct platform_driver qcom_nandc_driver = {
3072         .driver = {
3073                 .name = "qcom-nandc",
3074                 .of_match_table = qcom_nandc_of_match,
3075         },
3076         .probe   = qcom_nandc_probe,
3077         .remove  = qcom_nandc_remove,
3078 };
3079 module_platform_driver(qcom_nandc_driver);
3080
3081 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3082 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3083 MODULE_LICENSE("GPL v2");