drivers/mtd/nand/qcom_nandc.c

   1 /*
   2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   3  *
   4  * This software is licensed under the terms of the GNU General Public
   5  * License version 2, as published by the Free Software Foundation, and
   6  * may be copied, distributed, and modified under those terms.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/slab.h>
  16 #include <linux/bitops.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmaengine.h>
  19 #include <linux/module.h>
  20 #include <linux/mtd/nand.h>
  21 #include <linux/mtd/partitions.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/delay.h>
  25
  26 /* NANDc reg offsets */
  27 #define NAND_FLASH_CMD                  0x00
  28 #define NAND_ADDR0                      0x04
  29 #define NAND_ADDR1                      0x08
  30 #define NAND_FLASH_CHIP_SELECT          0x0c
  31 #define NAND_EXEC_CMD                   0x10
  32 #define NAND_FLASH_STATUS               0x14
  33 #define NAND_BUFFER_STATUS              0x18
  34 #define NAND_DEV0_CFG0                  0x20
  35 #define NAND_DEV0_CFG1                  0x24
  36 #define NAND_DEV0_ECC_CFG               0x28
  37 #define NAND_DEV1_ECC_CFG               0x2c
  38 #define NAND_DEV1_CFG0                  0x30
  39 #define NAND_DEV1_CFG1                  0x34
  40 #define NAND_READ_ID                    0x40
  41 #define NAND_READ_STATUS                0x44
  42 #define NAND_DEV_CMD0                   0xa0
  43 #define NAND_DEV_CMD1                   0xa4
  44 #define NAND_DEV_CMD2                   0xa8
  45 #define NAND_DEV_CMD_VLD                0xac
  46 #define SFLASHC_BURST_CFG               0xe0
  47 #define NAND_ERASED_CW_DETECT_CFG       0xe8
  48 #define NAND_ERASED_CW_DETECT_STATUS    0xec
  49 #define NAND_EBI2_ECC_BUF_CFG           0xf0
  50 #define FLASH_BUF_ACC                   0x100
  51
  52 #define NAND_CTRL                       0xf00
  53 #define NAND_VERSION                    0xf08
  54 #define NAND_READ_LOCATION_0            0xf20
  55 #define NAND_READ_LOCATION_1            0xf24
  56
  57 /* dummy register offsets, used by write_reg_dma */
  58 #define NAND_DEV_CMD1_RESTORE           0xdead
  59 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  60
  61 /* NAND_FLASH_CMD bits */
  62 #define PAGE_ACC                        BIT(4)
  63 #define LAST_PAGE                       BIT(5)
  64
  65 /* NAND_FLASH_CHIP_SELECT bits */
  66 #define NAND_DEV_SEL                    0
  67 #define DM_EN                           BIT(2)
  68
  69 /* NAND_FLASH_STATUS bits */
  70 #define FS_OP_ERR                       BIT(4)
  71 #define FS_READY_BSY_N                  BIT(5)
  72 #define FS_MPU_ERR                      BIT(8)
  73 #define FS_DEVICE_STS_ERR               BIT(16)
  74 #define FS_DEVICE_WP                    BIT(23)
  75
  76 /* NAND_BUFFER_STATUS bits */
  77 #define BS_UNCORRECTABLE_BIT            BIT(8)
  78 #define BS_CORRECTABLE_ERR_MSK          0x1f
  79
  80 /* NAND_DEVn_CFG0 bits */
  81 #define DISABLE_STATUS_AFTER_WRITE      4
  82 #define CW_PER_PAGE                     6
  83 #define UD_SIZE_BYTES                   9
  84 #define ECC_PARITY_SIZE_BYTES_RS        19
  85 #define SPARE_SIZE_BYTES                23
  86 #define NUM_ADDR_CYCLES                 27
  87 #define STATUS_BFR_READ                 30
  88 #define SET_RD_MODE_AFTER_STATUS        31
  89
  90 /* NAND_DEVn_CFG0 bits */
  91 #define DEV0_CFG1_ECC_DISABLE           0
  92 #define WIDE_FLASH                      1
  93 #define NAND_RECOVERY_CYCLES            2
  94 #define CS_ACTIVE_BSY                   5
  95 #define BAD_BLOCK_BYTE_NUM              6
  96 #define BAD_BLOCK_IN_SPARE_AREA         16
  97 #define WR_RD_BSY_GAP                   17
  98 #define ENABLE_BCH_ECC                  27
  99
 100 /* NAND_DEV0_ECC_CFG bits */
 101 #define ECC_CFG_ECC_DISABLE             0
 102 #define ECC_SW_RESET                    1
 103 #define ECC_MODE                        4
 104 #define ECC_PARITY_SIZE_BYTES_BCH       8
 105 #define ECC_NUM_DATA_BYTES              16
 106 #define ECC_FORCE_CLK_OPEN              30
 107
 108 /* NAND_DEV_CMD1 bits */
 109 #define READ_ADDR                       0
 110
 111 /* NAND_DEV_CMD_VLD bits */
 112 #define READ_START_VLD                  BIT(0)
 113 #define READ_STOP_VLD                   BIT(1)
 114 #define WRITE_START_VLD                 BIT(2)
 115 #define ERASE_START_VLD                 BIT(3)
 116 #define SEQ_READ_START_VLD              BIT(4)
 117
 118 /* NAND_EBI2_ECC_BUF_CFG bits */
 119 #define NUM_STEPS                       0
 120
 121 /* NAND_ERASED_CW_DETECT_CFG bits */
 122 #define ERASED_CW_ECC_MASK              1
 123 #define AUTO_DETECT_RES                 0
 124 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 125 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 126 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 127 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 128 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 129
 130 /* NAND_ERASED_CW_DETECT_STATUS bits */
 131 #define PAGE_ALL_ERASED                 BIT(7)
 132 #define CODEWORD_ALL_ERASED             BIT(6)
 133 #define PAGE_ERASED                     BIT(5)
 134 #define CODEWORD_ERASED                 BIT(4)
 135 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 136 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 137
 138 /* Version Mask */
 139 #define NAND_VERSION_MAJOR_MASK         0xf0000000
 140 #define NAND_VERSION_MAJOR_SHIFT        28
 141 #define NAND_VERSION_MINOR_MASK         0x0fff0000
 142 #define NAND_VERSION_MINOR_SHIFT        16
 143
 144 /* NAND OP_CMDs */
 145 #define OP_PAGE_READ                    0x2
 146 #define OP_PAGE_READ_WITH_ECC           0x3
 147 #define OP_PAGE_READ_WITH_ECC_SPARE     0x4
 148 #define OP_PROGRAM_PAGE                 0x6
 149 #define OP_PAGE_PROGRAM_WITH_ECC        0x7
 150 #define OP_PROGRAM_PAGE_SPARE           0x9
 151 #define OP_BLOCK_ERASE                  0xa
 152 #define OP_FETCH_ID                     0xb
 153 #define OP_RESET_DEVICE                 0xd
 154
 155 /* Default Value for NAND_DEV_CMD_VLD */
 156 #define NAND_DEV_CMD_VLD_VAL            (READ_START_VLD | WRITE_START_VLD | \
 157                                          ERASE_START_VLD | SEQ_READ_START_VLD)
 158
 159 /*
 160  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 161  * the driver calls the chunks 'step' or 'codeword' interchangeably
 162  */
 163 #define NANDC_STEP_SIZE                 512
 164
 165 /*
 166  * the largest page size we support is 8K, this will have 16 steps/codewords
 167  * of 512 bytes each
 168  */
 169 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 170
 171 /* we read at most 3 registers per codeword scan */
 172 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 173
 174 /* ECC modes supported by the controller */
 175 #define ECC_NONE        BIT(0)
 176 #define ECC_RS_4BIT     BIT(1)
 177 #define ECC_BCH_4BIT    BIT(2)
 178 #define ECC_BCH_8BIT    BIT(3)
 179
 180 struct desc_info {
 181         struct list_head node;
 182
 183         enum dma_data_direction dir;
 184         struct scatterlist sgl;
 185         struct dma_async_tx_descriptor *dma_desc;
 186 };
 187
 188 /*
 189  * holds the current register values that we want to write. acts as a contiguous
 190  * chunk of memory which we use to write the controller registers through DMA.
 191  */
 192 struct nandc_regs {
 193         __le32 cmd;
 194         __le32 addr0;
 195         __le32 addr1;
 196         __le32 chip_sel;
 197         __le32 exec;
 198
 199         __le32 cfg0;
 200         __le32 cfg1;
 201         __le32 ecc_bch_cfg;
 202
 203         __le32 clrflashstatus;
 204         __le32 clrreadstatus;
 205
 206         __le32 cmd1;
 207         __le32 vld;
 208
 209         __le32 orig_cmd1;
 210         __le32 orig_vld;
 211
 212         __le32 ecc_buf_cfg;
 213 };
 214
 215 /*
 216  * NAND controller data struct
 217  *
 218  * @controller:                 base controller structure
 219  * @host_list:                  list containing all the chips attached to the
 220  *                              controller
 221  * @dev:                        parent device
 222  * @base:                       MMIO base
 223  * @base_dma:                   physical base address of controller registers
 224  * @core_clk:                   controller clock
 225  * @aon_clk:                    another controller clock
 226  *
 227  * @chan:                       dma channel
 228  * @cmd_crci:                   ADM DMA CRCI for command flow control
 229  * @data_crci:                  ADM DMA CRCI for data flow control
 230  * @desc_list:                  DMA descriptor list (list of desc_infos)
 231  *
 232  * @data_buffer:                our local DMA buffer for page read/writes,
 233  *                              used when we can't use the buffer provided
 234  *                              by upper layers directly
 235  * @buf_size/count/start:       markers for chip->read_buf/write_buf functions
 236  * @reg_read_buf:               local buffer for reading back registers via DMA
 237  * @reg_read_pos:               marker for data read in reg_read_buf
 238  *
 239  * @regs:                       a contiguous chunk of memory for DMA register
 240  *                              writes. contains the register values to be
 241  *                              written to controller
 242  * @cmd1/vld:                   some fixed controller register values
 243  * @ecc_modes:                  supported ECC modes by the current controller,
 244  *                              initialized via DT match data
 245  */
 246 struct qcom_nand_controller {
 247         struct nand_hw_control controller;
 248         struct list_head host_list;
 249
 250         struct device *dev;
 251
 252         void __iomem *base;
 253         dma_addr_t base_dma;
 254
 255         struct clk *core_clk;
 256         struct clk *aon_clk;
 257
 258         struct dma_chan *chan;
 259         unsigned int cmd_crci;
 260         unsigned int data_crci;
 261         struct list_head desc_list;
 262
 263         u8              *data_buffer;
 264         int             buf_size;
 265         int             buf_count;
 266         int             buf_start;
 267
 268         __le32 *reg_read_buf;
 269         int reg_read_pos;
 270
 271         struct nandc_regs *regs;
 272
 273         u32 cmd1, vld;
 274         u32 ecc_modes;
 275 };
 276
 277 /*
 278  * NAND chip structure
 279  *
 280  * @chip:                       base NAND chip structure
 281  * @node:                       list node to add itself to host_list in
 282  *                              qcom_nand_controller
 283  *
 284  * @cs:                         chip select value for this chip
 285  * @cw_size:                    the number of bytes in a single step/codeword
 286  *                              of a page, consisting of all data, ecc, spare
 287  *                              and reserved bytes
 288  * @cw_data:                    the number of bytes within a codeword protected
 289  *                              by ECC
 290  * @use_ecc:                    request the controller to use ECC for the
 291  *                              upcoming read/write
 292  * @bch_enabled:                flag to tell whether BCH ECC mode is used
 293  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 294  *                              chip
 295  * @status:                     value to be returned if NAND_CMD_STATUS command
 296  *                              is executed
 297  * @last_command:               keeps track of last command on this chip. used
 298  *                              for reading correct status
 299  *
 300  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 301  *                              ecc/non-ecc mode for the current nand flash
 302  *                              device
 303  */
 304 struct qcom_nand_host {
 305         struct nand_chip chip;
 306         struct list_head node;
 307
 308         int cs;
 309         int cw_size;
 310         int cw_data;
 311         bool use_ecc;
 312         bool bch_enabled;
 313         int ecc_bytes_hw;
 314         int spare_bytes;
 315         int bbm_size;
 316         u8 status;
 317         int last_command;
 318
 319         u32 cfg0, cfg1;
 320         u32 cfg0_raw, cfg1_raw;
 321         u32 ecc_buf_cfg;
 322         u32 ecc_bch_cfg;
 323         u32 clrflashstatus;
 324         u32 clrreadstatus;
 325 };
 326
 327 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 328 {
 329         return container_of(chip, struct qcom_nand_host, chip);
 330 }
 331
 332 static inline struct qcom_nand_controller *
 333 get_qcom_nand_controller(struct nand_chip *chip)
 334 {
 335         return container_of(chip->controller, struct qcom_nand_controller,
 336                             controller);
 337 }
 338
 339 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 340 {
 341         return ioread32(nandc->base + offset);
 342 }
 343
 344 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 345                                u32 val)
 346 {
 347         iowrite32(val, nandc->base + offset);
 348 }
 349
 350 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 351 {
 352         switch (offset) {
 353         case NAND_FLASH_CMD:
 354                 return &regs->cmd;
 355         case NAND_ADDR0:
 356                 return &regs->addr0;
 357         case NAND_ADDR1:
 358                 return &regs->addr1;
 359         case NAND_FLASH_CHIP_SELECT:
 360                 return &regs->chip_sel;
 361         case NAND_EXEC_CMD:
 362                 return &regs->exec;
 363         case NAND_FLASH_STATUS:
 364                 return &regs->clrflashstatus;
 365         case NAND_DEV0_CFG0:
 366                 return &regs->cfg0;
 367         case NAND_DEV0_CFG1:
 368                 return &regs->cfg1;
 369         case NAND_DEV0_ECC_CFG:
 370                 return &regs->ecc_bch_cfg;
 371         case NAND_READ_STATUS:
 372                 return &regs->clrreadstatus;
 373         case NAND_DEV_CMD1:
 374                 return &regs->cmd1;
 375         case NAND_DEV_CMD1_RESTORE:
 376                 return &regs->orig_cmd1;
 377         case NAND_DEV_CMD_VLD:
 378                 return &regs->vld;
 379         case NAND_DEV_CMD_VLD_RESTORE:
 380                 return &regs->orig_vld;
 381         case NAND_EBI2_ECC_BUF_CFG:
 382                 return &regs->ecc_buf_cfg;
 383         default:
 384                 return NULL;
 385         }
 386 }
 387
 388 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
 389                           u32 val)
 390 {
 391         struct nandc_regs *regs = nandc->regs;
 392         __le32 *reg;
 393
 394         reg = offset_to_nandc_reg(regs, offset);
 395
 396         if (reg)
 397                 *reg = cpu_to_le32(val);
 398 }
 399
 400 /* helper to configure address register values */
 401 static void set_address(struct qcom_nand_host *host, u16 column, int page)
 402 {
 403         struct nand_chip *chip = &host->chip;
 404         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 405
 406         if (chip->options & NAND_BUSWIDTH_16)
 407                 column >>= 1;
 408
 409         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
 410         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
 411 }
 412
 413 /*
 414  * update_rw_regs:      set up read/write register values, these will be
 415  *                      written to the NAND controller registers via DMA
 416  *
 417  * @num_cw:             number of steps for the read/write operation
 418  * @read:               read or write operation
 419  */
 420 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 421 {
 422         struct nand_chip *chip = &host->chip;
 423         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 424         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 425
 426         if (read) {
 427                 if (host->use_ecc)
 428                         cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 429                 else
 430                         cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
 431         } else {
 432                 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 433         }
 434
 435         if (host->use_ecc) {
 436                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 437                                 (num_cw - 1) << CW_PER_PAGE;
 438
 439                 cfg1 = host->cfg1;
 440                 ecc_bch_cfg = host->ecc_bch_cfg;
 441         } else {
 442                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 443                                 (num_cw - 1) << CW_PER_PAGE;
 444
 445                 cfg1 = host->cfg1_raw;
 446                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 447         }
 448
 449         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
 450         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
 451         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
 452         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 453         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 454         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 455         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 456         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 457 }
 458
 459 static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
 460                          int reg_off, const void *vaddr, int size,
 461                          bool flow_control)
 462 {
 463         struct desc_info *desc;
 464         struct dma_async_tx_descriptor *dma_desc;
 465         struct scatterlist *sgl;
 466         struct dma_slave_config slave_conf;
 467         enum dma_transfer_direction dir_eng;
 468         int ret;
 469
 470         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 471         if (!desc)
 472                 return -ENOMEM;
 473
 474         sgl = &desc->sgl;
 475
 476         sg_init_one(sgl, vaddr, size);
 477
 478         if (read) {
 479                 dir_eng = DMA_DEV_TO_MEM;
 480                 desc->dir = DMA_FROM_DEVICE;
 481         } else {
 482                 dir_eng = DMA_MEM_TO_DEV;
 483                 desc->dir = DMA_TO_DEVICE;
 484         }
 485
 486         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 487         if (ret == 0) {
 488                 ret = -ENOMEM;
 489                 goto err;
 490         }
 491
 492         memset(&slave_conf, 0x00, sizeof(slave_conf));
 493
 494         slave_conf.device_fc = flow_control;
 495         if (read) {
 496                 slave_conf.src_maxburst = 16;
 497                 slave_conf.src_addr = nandc->base_dma + reg_off;
 498                 slave_conf.slave_id = nandc->data_crci;
 499         } else {
 500                 slave_conf.dst_maxburst = 16;
 501                 slave_conf.dst_addr = nandc->base_dma + reg_off;
 502                 slave_conf.slave_id = nandc->cmd_crci;
 503         }
 504
 505         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 506         if (ret) {
 507                 dev_err(nandc->dev, "failed to configure dma channel\n");
 508                 goto err;
 509         }
 510
 511         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
 512         if (!dma_desc) {
 513                 dev_err(nandc->dev, "failed to prepare desc\n");
 514                 ret = -EINVAL;
 515                 goto err;
 516         }
 517
 518         desc->dma_desc = dma_desc;
 519
 520         list_add_tail(&desc->node, &nandc->desc_list);
 521
 522         return 0;
 523 err:
 524         kfree(desc);
 525
 526         return ret;
 527 }
 528
 529 /*
 530  * read_reg_dma:        prepares a descriptor to read a given number of
 531  *                      contiguous registers to the reg_read_buf pointer
 532  *
 533  * @first:              offset of the first register in the contiguous block
 534  * @num_regs:           number of registers to read
 535  */
 536 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 537                         int num_regs)
 538 {
 539         bool flow_control = false;
 540         void *vaddr;
 541         int size;
 542
 543         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 544                 flow_control = true;
 545
 546         size = num_regs * sizeof(u32);
 547         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 548         nandc->reg_read_pos += num_regs;
 549
 550         return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
 551 }
 552
 553 /*
 554  * write_reg_dma:       prepares a descriptor to write a given number of
 555  *                      contiguous registers
 556  *
 557  * @first:              offset of the first register in the contiguous block
 558  * @num_regs:           number of registers to write
 559  */
 560 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
 561                          int num_regs)
 562 {
 563         bool flow_control = false;
 564         struct nandc_regs *regs = nandc->regs;
 565         void *vaddr;
 566         int size;
 567
 568         vaddr = offset_to_nandc_reg(regs, first);
 569
 570         if (first == NAND_FLASH_CMD)
 571                 flow_control = true;
 572
 573         if (first == NAND_DEV_CMD1_RESTORE)
 574                 first = NAND_DEV_CMD1;
 575
 576         if (first == NAND_DEV_CMD_VLD_RESTORE)
 577                 first = NAND_DEV_CMD_VLD;
 578
 579         size = num_regs * sizeof(u32);
 580
 581         return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
 582 }
 583
 584 /*
 585  * read_data_dma:       prepares a DMA descriptor to transfer data from the
 586  *                      controller's internal buffer to the buffer 'vaddr'
 587  *
 588  * @reg_off:            offset within the controller's data buffer
 589  * @vaddr:              virtual address of the buffer we want to write to
 590  * @size:               DMA transaction size in bytes
 591  */
 592 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 593                          const u8 *vaddr, int size)
 594 {
 595         return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
 596 }
 597
 598 /*
 599  * write_data_dma:      prepares a DMA descriptor to transfer data from
 600  *                      'vaddr' to the controller's internal buffer
 601  *
 602  * @reg_off:            offset within the controller's data buffer
 603  * @vaddr:              virtual address of the buffer we want to read from
 604  * @size:               DMA transaction size in bytes
 605  */
 606 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 607                           const u8 *vaddr, int size)
 608 {
 609         return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
 610 }
 611
 612 /*
 613  * helper to prepare dma descriptors to configure registers needed for reading a
 614  * codeword/step in a page
 615  */
 616 static void config_cw_read(struct qcom_nand_controller *nandc)
 617 {
 618         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 619         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
 620         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
 621
 622         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 623
 624         read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
 625         read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
 626 }
 627
 628 /*
 629  * helpers to prepare dma descriptors used to configure registers needed for
 630  * writing a codeword/step in a page
 631  */
 632 static void config_cw_write_pre(struct qcom_nand_controller *nandc)
 633 {
 634         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 635         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
 636         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
 637 }
 638
 639 static void config_cw_write_post(struct qcom_nand_controller *nandc)
 640 {
 641         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 642
 643         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 644
 645         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 646         write_reg_dma(nandc, NAND_READ_STATUS, 1);
 647 }
 648
 649 /*
 650  * the following functions are used within chip->cmdfunc() to perform different
 651  * NAND_CMD_* commands
 652  */
 653
 654 /* sets up descriptors for NAND_CMD_PARAM */
 655 static int nandc_param(struct qcom_nand_host *host)
 656 {
 657         struct nand_chip *chip = &host->chip;
 658         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 659
 660         /*
 661          * NAND_CMD_PARAM is called before we know much about the FLASH chip
 662          * in use. we configure the controller to perform a raw read of 512
 663          * bytes to read onfi params
 664          */
 665         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
 666         nandc_set_reg(nandc, NAND_ADDR0, 0);
 667         nandc_set_reg(nandc, NAND_ADDR1, 0);
 668         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
 669                                         | 512 << UD_SIZE_BYTES
 670                                         | 5 << NUM_ADDR_CYCLES
 671                                         | 0 << SPARE_SIZE_BYTES);
 672         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
 673                                         | 0 << CS_ACTIVE_BSY
 674                                         | 17 << BAD_BLOCK_BYTE_NUM
 675                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
 676                                         | 2 << WR_RD_BSY_GAP
 677                                         | 0 << WIDE_FLASH
 678                                         | 1 << DEV0_CFG1_ECC_DISABLE);
 679         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
 680
 681         /* configure CMD1 and VLD for ONFI param probing */
 682         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
 683                       (nandc->vld & ~READ_START_VLD));
 684         nandc_set_reg(nandc, NAND_DEV_CMD1,
 685                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
 686                       | NAND_CMD_PARAM << READ_ADDR);
 687
 688         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 689
 690         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
 691         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
 692
 693         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
 694         write_reg_dma(nandc, NAND_DEV_CMD1, 1);
 695
 696         nandc->buf_count = 512;
 697         memset(nandc->data_buffer, 0xff, nandc->buf_count);
 698
 699         config_cw_read(nandc);
 700
 701         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
 702                       nandc->buf_count);
 703
 704         /* restore CMD1 and VLD regs */
 705         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
 706         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);
 707
 708         return 0;
 709 }
 710
 711 /* sets up descriptors for NAND_CMD_ERASE1 */
 712 static int erase_block(struct qcom_nand_host *host, int page_addr)
 713 {
 714         struct nand_chip *chip = &host->chip;
 715         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 716
 717         nandc_set_reg(nandc, NAND_FLASH_CMD,
 718                       OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
 719         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
 720         nandc_set_reg(nandc, NAND_ADDR1, 0);
 721         nandc_set_reg(nandc, NAND_DEV0_CFG0,
 722                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
 723         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
 724         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 725         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 726         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 727
 728         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 729         write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
 730         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 731
 732         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 733
 734         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 735         write_reg_dma(nandc, NAND_READ_STATUS, 1);
 736
 737         return 0;
 738 }
 739
 740 /* sets up descriptors for NAND_CMD_READID */
 741 static int read_id(struct qcom_nand_host *host, int column)
 742 {
 743         struct nand_chip *chip = &host->chip;
 744         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 745
 746         if (column == -1)
 747                 return 0;
 748
 749         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
 750         nandc_set_reg(nandc, NAND_ADDR0, column);
 751         nandc_set_reg(nandc, NAND_ADDR1, 0);
 752         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
 753         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 754
 755         write_reg_dma(nandc, NAND_FLASH_CMD, 4);
 756         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 757
 758         read_reg_dma(nandc, NAND_READ_ID, 1);
 759
 760         return 0;
 761 }
 762
 763 /* sets up descriptors for NAND_CMD_RESET */
 764 static int reset(struct qcom_nand_host *host)
 765 {
 766         struct nand_chip *chip = &host->chip;
 767         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 768
 769         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
 770         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 771
 772         write_reg_dma(nandc, NAND_FLASH_CMD, 1);
 773         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 774
 775         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 776
 777         return 0;
 778 }
 779
 780 /* helpers to submit/free our list of dma descriptors */
 781 static int submit_descs(struct qcom_nand_controller *nandc)
 782 {
 783         struct desc_info *desc;
 784         dma_cookie_t cookie = 0;
 785
 786         list_for_each_entry(desc, &nandc->desc_list, node)
 787                 cookie = dmaengine_submit(desc->dma_desc);
 788
 789         if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
 790                 return -ETIMEDOUT;
 791
 792         return 0;
 793 }
 794
 795 static void free_descs(struct qcom_nand_controller *nandc)
 796 {
 797         struct desc_info *desc, *n;
 798
 799         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
 800                 list_del(&desc->node);
 801                 dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
 802                 kfree(desc);
 803         }
 804 }
 805
 806 /* reset the register read buffer for next NAND operation */
 807 static void clear_read_regs(struct qcom_nand_controller *nandc)
 808 {
 809         nandc->reg_read_pos = 0;
 810         memset(nandc->reg_read_buf, 0,
 811                MAX_REG_RD * sizeof(*nandc->reg_read_buf));
 812 }
 813
 814 static void pre_command(struct qcom_nand_host *host, int command)
 815 {
 816         struct nand_chip *chip = &host->chip;
 817         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 818
 819         nandc->buf_count = 0;
 820         nandc->buf_start = 0;
 821         host->use_ecc = false;
 822         host->last_command = command;
 823
 824         clear_read_regs(nandc);
 825 }
 826
 827 /*
 828  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
 829  * privately maintained status byte, this status byte can be read after
 830  * NAND_CMD_STATUS is called
 831  */
 832 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
 833 {
 834         struct nand_chip *chip = &host->chip;
 835         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 836         struct nand_ecc_ctrl *ecc = &chip->ecc;
 837         int num_cw;
 838         int i;
 839
 840         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
 841
 842         for (i = 0; i < num_cw; i++) {
 843                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
 844
 845                 if (flash_status & FS_MPU_ERR)
 846                         host->status &= ~NAND_STATUS_WP;
 847
 848                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
 849                                                  (flash_status &
 850                                                   FS_DEVICE_STS_ERR)))
 851                         host->status |= NAND_STATUS_FAIL;
 852         }
 853 }
 854
 855 static void post_command(struct qcom_nand_host *host, int command)
 856 {
 857         struct nand_chip *chip = &host->chip;
 858         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 859
 860         switch (command) {
 861         case NAND_CMD_READID:
 862                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
 863                        nandc->buf_count);
 864                 break;
 865         case NAND_CMD_PAGEPROG:
 866         case NAND_CMD_ERASE1:
 867                 parse_erase_write_errors(host, command);
 868                 break;
 869         default:
 870                 break;
 871         }
 872 }
 873
 874 /*
 875  * Implements chip->cmdfunc. It's  only used for a limited set of commands.
 876  * The rest of the commands wouldn't be called by upper layers. For example,
 877  * NAND_CMD_READOOB would never be called because we have our own versions
 878  * of read_oob ops for nand_ecc_ctrl.
 879  */
 880 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
 881                                int column, int page_addr)
 882 {
 883         struct nand_chip *chip = mtd_to_nand(mtd);
 884         struct qcom_nand_host *host = to_qcom_nand_host(chip);
 885         struct nand_ecc_ctrl *ecc = &chip->ecc;
 886         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 887         bool wait = false;
 888         int ret = 0;
 889
 890         pre_command(host, command);
 891
 892         switch (command) {
 893         case NAND_CMD_RESET:
 894                 ret = reset(host);
 895                 wait = true;
 896                 break;
 897
 898         case NAND_CMD_READID:
 899                 nandc->buf_count = 4;
 900                 ret = read_id(host, column);
 901                 wait = true;
 902                 break;
 903
 904         case NAND_CMD_PARAM:
 905                 ret = nandc_param(host);
 906                 wait = true;
 907                 break;
 908
 909         case NAND_CMD_ERASE1:
 910                 ret = erase_block(host, page_addr);
 911                 wait = true;
 912                 break;
 913
 914         case NAND_CMD_READ0:
 915                 /* we read the entire page for now */
 916                 WARN_ON(column != 0);
 917
 918                 host->use_ecc = true;
 919                 set_address(host, 0, page_addr);
 920                 update_rw_regs(host, ecc->steps, true);
 921                 break;
 922
 923         case NAND_CMD_SEQIN:
 924                 WARN_ON(column != 0);
 925                 set_address(host, 0, page_addr);
 926                 break;
 927
 928         case NAND_CMD_PAGEPROG:
 929         case NAND_CMD_STATUS:
 930         case NAND_CMD_NONE:
 931         default:
 932                 break;
 933         }
 934
 935         if (ret) {
 936                 dev_err(nandc->dev, "failure executing command %d\n",
 937                         command);
 938                 free_descs(nandc);
 939                 return;
 940         }
 941
 942         if (wait) {
 943                 ret = submit_descs(nandc);
 944                 if (ret)
 945                         dev_err(nandc->dev,
 946                                 "failure submitting descs for command %d\n",
 947                                 command);
 948         }
 949
 950         free_descs(nandc);
 951
 952         post_command(host, command);
 953 }
 954
 955 /*
 956  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
 957  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
 958  *
 959  * when using RS ECC, the HW reports the same erros when reading an erased CW,
 960  * but it notifies that it is an erased CW by placing special characters at
 961  * certain offsets in the buffer.
 962  *
 963  * verify if the page is erased or not, and fix up the page for RS ECC by
 964  * replacing the special characters with 0xff.
 965  */
 966 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
 967 {
 968         u8 empty1, empty2;
 969
 970         /*
 971          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
 972          * is erased by looking for 0x54s at offsets 3 and 175 from the
 973          * beginning of each codeword
 974          */
 975
 976         empty1 = data_buf[3];
 977         empty2 = data_buf[175];
 978
 979         /*
 980          * if the erased codework markers, if they exist override them with
 981          * 0xffs
 982          */
 983         if ((empty1 == 0x54 && empty2 == 0xff) ||
 984             (empty1 == 0xff && empty2 == 0x54)) {
 985                 data_buf[3] = 0xff;
 986                 data_buf[175] = 0xff;
 987         }
 988
 989         /*
 990          * check if the entire chunk contains 0xffs or not. if it doesn't, then
 991          * restore the original values at the special offsets
 992          */
 993         if (memchr_inv(data_buf, 0xff, data_len)) {
 994                 data_buf[3] = empty1;
 995                 data_buf[175] = empty2;
 996
 997                 return false;
 998         }
 999
1000         return true;
1001 }
1002
1003 struct read_stats {
1004         __le32 flash;
1005         __le32 buffer;
1006         __le32 erased_cw;
1007 };
1008
1009 /*
1010  * reads back status registers set by the controller to notify page read
1011  * errors. this is equivalent to what 'ecc->correct()' would do.
1012  */
1013 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1014                              u8 *oob_buf)
1015 {
1016         struct nand_chip *chip = &host->chip;
1017         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1018         struct mtd_info *mtd = nand_to_mtd(chip);
1019         struct nand_ecc_ctrl *ecc = &chip->ecc;
1020         unsigned int max_bitflips = 0;
1021         struct read_stats *buf;
1022         int i;
1023
1024         buf = (struct read_stats *)nandc->reg_read_buf;
1025
1026         for (i = 0; i < ecc->steps; i++, buf++) {
1027                 u32 flash, buffer, erased_cw;
1028                 int data_len, oob_len;
1029
1030                 if (i == (ecc->steps - 1)) {
1031                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1032                         oob_len = ecc->steps << 2;
1033                 } else {
1034                         data_len = host->cw_data;
1035                         oob_len = 0;
1036                 }
1037
1038                 flash = le32_to_cpu(buf->flash);
1039                 buffer = le32_to_cpu(buf->buffer);
1040                 erased_cw = le32_to_cpu(buf->erased_cw);
1041
1042                 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1043                         bool erased;
1044
1045                         /* ignore erased codeword errors */
1046                         if (host->bch_enabled) {
1047                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1048                                          true : false;
1049                         } else {
1050                                 erased = erased_chunk_check_and_fixup(data_buf,
1051                                                                       data_len);
1052                         }
1053
1054                         if (erased) {
1055                                 data_buf += data_len;
1056                                 if (oob_buf)
1057                                         oob_buf += oob_len + ecc->bytes;
1058                                 continue;
1059                         }
1060
1061                         if (buffer & BS_UNCORRECTABLE_BIT) {
1062                                 int ret, ecclen, extraooblen;
1063                                 void *eccbuf;
1064
1065                                 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1066                                 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1067                                 extraooblen = oob_buf ? oob_len : 0;
1068
1069                                 /*
1070                                  * make sure it isn't an erased page reported
1071                                  * as not-erased by HW because of a few bitflips
1072                                  */
1073                                 ret = nand_check_erased_ecc_chunk(data_buf,
1074                                         data_len, eccbuf, ecclen, oob_buf,
1075                                         extraooblen, ecc->strength);
1076                                 if (ret < 0) {
1077                                         mtd->ecc_stats.failed++;
1078                                 } else {
1079                                         mtd->ecc_stats.corrected += ret;
1080                                         max_bitflips =
1081                                                 max_t(unsigned int, max_bitflips, ret);
1082                                 }
1083                         }
1084                 } else {
1085                         unsigned int stat;
1086
1087                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1088                         mtd->ecc_stats.corrected += stat;
1089                         max_bitflips = max(max_bitflips, stat);
1090                 }
1091
1092                 data_buf += data_len;
1093                 if (oob_buf)
1094                         oob_buf += oob_len + ecc->bytes;
1095         }
1096
1097         return max_bitflips;
1098 }
1099
1100 /*
1101  * helper to perform the actual page read operation, used by ecc->read_page(),
1102  * ecc->read_oob()
1103  */
1104 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1105                          u8 *oob_buf)
1106 {
1107         struct nand_chip *chip = &host->chip;
1108         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1109         struct nand_ecc_ctrl *ecc = &chip->ecc;
1110         int i, ret;
1111
1112         /* queue cmd descs for each codeword */
1113         for (i = 0; i < ecc->steps; i++) {
1114                 int data_size, oob_size;
1115
1116                 if (i == (ecc->steps - 1)) {
1117                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1118                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1119                                    host->spare_bytes;
1120                 } else {
1121                         data_size = host->cw_data;
1122                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1123                 }
1124
1125                 config_cw_read(nandc);
1126
1127                 if (data_buf)
1128                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1129                                       data_size);
1130
1131                 /*
1132                  * when ecc is enabled, the controller doesn't read the real
1133                  * or dummy bad block markers in each chunk. To maintain a
1134                  * consistent layout across RAW and ECC reads, we just
1135                  * leave the real/dummy BBM offsets empty (i.e, filled with
1136                  * 0xffs)
1137                  */
1138                 if (oob_buf) {
1139                         int j;
1140
1141                         for (j = 0; j < host->bbm_size; j++)
1142                                 *oob_buf++ = 0xff;
1143
1144                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1145                                       oob_buf, oob_size);
1146                 }
1147
1148                 if (data_buf)
1149                         data_buf += data_size;
1150                 if (oob_buf)
1151                         oob_buf += oob_size;
1152         }
1153
1154         ret = submit_descs(nandc);
1155         if (ret)
1156                 dev_err(nandc->dev, "failure to read page/oob\n");
1157
1158         free_descs(nandc);
1159
1160         return ret;
1161 }
1162
1163 /*
1164  * a helper that copies the last step/codeword of a page (containing free oob)
1165  * into our local buffer
1166  */
1167 static int copy_last_cw(struct qcom_nand_host *host, int page)
1168 {
1169         struct nand_chip *chip = &host->chip;
1170         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1171         struct nand_ecc_ctrl *ecc = &chip->ecc;
1172         int size;
1173         int ret;
1174
1175         clear_read_regs(nandc);
1176
1177         size = host->use_ecc ? host->cw_data : host->cw_size;
1178
1179         /* prepare a clean read buffer */
1180         memset(nandc->data_buffer, 0xff, size);
1181
1182         set_address(host, host->cw_size * (ecc->steps - 1), page);
1183         update_rw_regs(host, 1, true);
1184
1185         config_cw_read(nandc);
1186
1187         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);
1188
1189         ret = submit_descs(nandc);
1190         if (ret)
1191                 dev_err(nandc->dev, "failed to copy last codeword\n");
1192
1193         free_descs(nandc);
1194
1195         return ret;
1196 }
1197
1198 /* implements ecc->read_page() */
1199 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1200                                 uint8_t *buf, int oob_required, int page)
1201 {
1202         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1203         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1204         u8 *data_buf, *oob_buf = NULL;
1205         int ret;
1206
1207         data_buf = buf;
1208         oob_buf = oob_required ? chip->oob_poi : NULL;
1209
1210         ret = read_page_ecc(host, data_buf, oob_buf);
1211         if (ret) {
1212                 dev_err(nandc->dev, "failure to read page\n");
1213                 return ret;
1214         }
1215
1216         return parse_read_errors(host, data_buf, oob_buf);
1217 }
1218
1219 /* implements ecc->read_page_raw() */
1220 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1221                                     struct nand_chip *chip, uint8_t *buf,
1222                                     int oob_required, int page)
1223 {
1224         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1225         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1226         u8 *data_buf, *oob_buf;
1227         struct nand_ecc_ctrl *ecc = &chip->ecc;
1228         int i, ret;
1229
1230         data_buf = buf;
1231         oob_buf = chip->oob_poi;
1232
1233         host->use_ecc = false;
1234         update_rw_regs(host, ecc->steps, true);
1235
1236         for (i = 0; i < ecc->steps; i++) {
1237                 int data_size1, data_size2, oob_size1, oob_size2;
1238                 int reg_off = FLASH_BUF_ACC;
1239
1240                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1241                 oob_size1 = host->bbm_size;
1242
1243                 if (i == (ecc->steps - 1)) {
1244                         data_size2 = ecc->size - data_size1 -
1245                                      ((ecc->steps - 1) << 2);
1246                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1247                                     host->spare_bytes;
1248                 } else {
1249                         data_size2 = host->cw_data - data_size1;
1250                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1251                 }
1252
1253                 config_cw_read(nandc);
1254
1255                 read_data_dma(nandc, reg_off, data_buf, data_size1);
1256                 reg_off += data_size1;
1257                 data_buf += data_size1;
1258
1259                 read_data_dma(nandc, reg_off, oob_buf, oob_size1);
1260                 reg_off += oob_size1;
1261                 oob_buf += oob_size1;
1262
1263                 read_data_dma(nandc, reg_off, data_buf, data_size2);
1264                 reg_off += data_size2;
1265                 data_buf += data_size2;
1266
1267                 read_data_dma(nandc, reg_off, oob_buf, oob_size2);
1268                 oob_buf += oob_size2;
1269         }
1270
1271         ret = submit_descs(nandc);
1272         if (ret)
1273                 dev_err(nandc->dev, "failure to read raw page\n");
1274
1275         free_descs(nandc);
1276
1277         return 0;
1278 }
1279
1280 /* implements ecc->read_oob() */
1281 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1282                                int page)
1283 {
1284         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1285         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1286         struct nand_ecc_ctrl *ecc = &chip->ecc;
1287         int ret;
1288
1289         clear_read_regs(nandc);
1290
1291         host->use_ecc = true;
1292         set_address(host, 0, page);
1293         update_rw_regs(host, ecc->steps, true);
1294
1295         ret = read_page_ecc(host, NULL, chip->oob_poi);
1296         if (ret)
1297                 dev_err(nandc->dev, "failure to read oob\n");
1298
1299         return ret;
1300 }
1301
1302 /* implements ecc->write_page() */
1303 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1304                                  const uint8_t *buf, int oob_required, int page)
1305 {
1306         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1307         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1308         struct nand_ecc_ctrl *ecc = &chip->ecc;
1309         u8 *data_buf, *oob_buf;
1310         int i, ret;
1311
1312         clear_read_regs(nandc);
1313
1314         data_buf = (u8 *)buf;
1315         oob_buf = chip->oob_poi;
1316
1317         host->use_ecc = true;
1318         update_rw_regs(host, ecc->steps, false);
1319
1320         for (i = 0; i < ecc->steps; i++) {
1321                 int data_size, oob_size;
1322
1323                 if (i == (ecc->steps - 1)) {
1324                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1325                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1326                                    host->spare_bytes;
1327                 } else {
1328                         data_size = host->cw_data;
1329                         oob_size = ecc->bytes;
1330                 }
1331
1332                 config_cw_write_pre(nandc);
1333
1334                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);
1335
1336                 /*
1337                  * when ECC is enabled, we don't really need to write anything
1338                  * to oob for the first n - 1 codewords since these oob regions
1339                  * just contain ECC bytes that's written by the controller
1340                  * itself. For the last codeword, we skip the bbm positions and
1341                  * write to the free oob area.
1342                  */
1343                 if (i == (ecc->steps - 1)) {
1344                         oob_buf += host->bbm_size;
1345
1346                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1347                                        oob_buf, oob_size);
1348                 }
1349
1350                 config_cw_write_post(nandc);
1351
1352                 data_buf += data_size;
1353                 oob_buf += oob_size;
1354         }
1355
1356         ret = submit_descs(nandc);
1357         if (ret)
1358                 dev_err(nandc->dev, "failure to write page\n");
1359
1360         free_descs(nandc);
1361
1362         return ret;
1363 }
1364
1365 /* implements ecc->write_page_raw() */
1366 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1367                                      struct nand_chip *chip, const uint8_t *buf,
1368                                      int oob_required, int page)
1369 {
1370         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1371         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1372         struct nand_ecc_ctrl *ecc = &chip->ecc;
1373         u8 *data_buf, *oob_buf;
1374         int i, ret;
1375
1376         clear_read_regs(nandc);
1377
1378         data_buf = (u8 *)buf;
1379         oob_buf = chip->oob_poi;
1380
1381         host->use_ecc = false;
1382         update_rw_regs(host, ecc->steps, false);
1383
1384         for (i = 0; i < ecc->steps; i++) {
1385                 int data_size1, data_size2, oob_size1, oob_size2;
1386                 int reg_off = FLASH_BUF_ACC;
1387
1388                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1389                 oob_size1 = host->bbm_size;
1390
1391                 if (i == (ecc->steps - 1)) {
1392                         data_size2 = ecc->size - data_size1 -
1393                                      ((ecc->steps - 1) << 2);
1394                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1395                                     host->spare_bytes;
1396                 } else {
1397                         data_size2 = host->cw_data - data_size1;
1398                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1399                 }
1400
1401                 config_cw_write_pre(nandc);
1402
1403                 write_data_dma(nandc, reg_off, data_buf, data_size1);
1404                 reg_off += data_size1;
1405                 data_buf += data_size1;
1406
1407                 write_data_dma(nandc, reg_off, oob_buf, oob_size1);
1408                 reg_off += oob_size1;
1409                 oob_buf += oob_size1;
1410
1411                 write_data_dma(nandc, reg_off, data_buf, data_size2);
1412                 reg_off += data_size2;
1413                 data_buf += data_size2;
1414
1415                 write_data_dma(nandc, reg_off, oob_buf, oob_size2);
1416                 oob_buf += oob_size2;
1417
1418                 config_cw_write_post(nandc);
1419         }
1420
1421         ret = submit_descs(nandc);
1422         if (ret)
1423                 dev_err(nandc->dev, "failure to write raw page\n");
1424
1425         free_descs(nandc);
1426
1427         return ret;
1428 }
1429
1430 /*
1431  * implements ecc->write_oob()
1432  *
1433  * the NAND controller cannot write only data or only oob within a codeword,
1434  * since ecc is calculated for the combined codeword. we first copy the
1435  * entire contents for the last codeword(data + oob), replace the old oob
1436  * with the new one in chip->oob_poi, and then write the entire codeword.
1437  * this read-copy-write operation results in a slight performance loss.
1438  */
1439 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1440                                 int page)
1441 {
1442         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1443         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1444         struct nand_ecc_ctrl *ecc = &chip->ecc;
1445         u8 *oob = chip->oob_poi;
1446         int data_size, oob_size;
1447         int ret, status = 0;
1448
1449         host->use_ecc = true;
1450
1451         ret = copy_last_cw(host, page);
1452         if (ret)
1453                 return ret;
1454
1455         clear_read_regs(nandc);
1456
1457         /* calculate the data and oob size for the last codeword/step */
1458         data_size = ecc->size - ((ecc->steps - 1) << 2);
1459         oob_size = mtd->oobavail;
1460
1461         /* override new oob content to last codeword */
1462         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1463                                     0, mtd->oobavail);
1464
1465         set_address(host, host->cw_size * (ecc->steps - 1), page);
1466         update_rw_regs(host, 1, false);
1467
1468         config_cw_write_pre(nandc);
1469         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1470                        data_size + oob_size);
1471         config_cw_write_post(nandc);
1472
1473         ret = submit_descs(nandc);
1474
1475         free_descs(nandc);
1476
1477         if (ret) {
1478                 dev_err(nandc->dev, "failure to write oob\n");
1479                 return -EIO;
1480         }
1481
1482         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1483
1484         status = chip->waitfunc(mtd, chip);
1485
1486         return status & NAND_STATUS_FAIL ? -EIO : 0;
1487 }
1488
1489 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1490 {
1491         struct nand_chip *chip = mtd_to_nand(mtd);
1492         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1493         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1494         struct nand_ecc_ctrl *ecc = &chip->ecc;
1495         int page, ret, bbpos, bad = 0;
1496         u32 flash_status;
1497
1498         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1499
1500         /*
1501          * configure registers for a raw sub page read, the address is set to
1502          * the beginning of the last codeword, we don't care about reading ecc
1503          * portion of oob. we just want the first few bytes from this codeword
1504          * that contains the BBM
1505          */
1506         host->use_ecc = false;
1507
1508         ret = copy_last_cw(host, page);
1509         if (ret)
1510                 goto err;
1511
1512         flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1513
1514         if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1515                 dev_warn(nandc->dev, "error when trying to read BBM\n");
1516                 goto err;
1517         }
1518
1519         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1520
1521         bad = nandc->data_buffer[bbpos] != 0xff;
1522
1523         if (chip->options & NAND_BUSWIDTH_16)
1524                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1525 err:
1526         return bad;
1527 }
1528
1529 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1530 {
1531         struct nand_chip *chip = mtd_to_nand(mtd);
1532         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1533         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1534         struct nand_ecc_ctrl *ecc = &chip->ecc;
1535         int page, ret, status = 0;
1536
1537         clear_read_regs(nandc);
1538
1539         /*
1540          * to mark the BBM as bad, we flash the entire last codeword with 0s.
1541          * we don't care about the rest of the content in the codeword since
1542          * we aren't going to use this block again
1543          */
1544         memset(nandc->data_buffer, 0x00, host->cw_size);
1545
1546         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1547
1548         /* prepare write */
1549         host->use_ecc = false;
1550         set_address(host, host->cw_size * (ecc->steps - 1), page);
1551         update_rw_regs(host, 1, false);
1552
1553         config_cw_write_pre(nandc);
1554         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
1555         config_cw_write_post(nandc);
1556
1557         ret = submit_descs(nandc);
1558
1559         free_descs(nandc);
1560
1561         if (ret) {
1562                 dev_err(nandc->dev, "failure to update BBM\n");
1563                 return -EIO;
1564         }
1565
1566         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1567
1568         status = chip->waitfunc(mtd, chip);
1569
1570         return status & NAND_STATUS_FAIL ? -EIO : 0;
1571 }
1572
1573 /*
1574  * the three functions below implement chip->read_byte(), chip->read_buf()
1575  * and chip->write_buf() respectively. these aren't used for
1576  * reading/writing page data, they are used for smaller data like reading
1577  * id, status etc
1578  */
1579 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
1580 {
1581         struct nand_chip *chip = mtd_to_nand(mtd);
1582         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1583         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1584         u8 *buf = nandc->data_buffer;
1585         u8 ret = 0x0;
1586
1587         if (host->last_command == NAND_CMD_STATUS) {
1588                 ret = host->status;
1589
1590                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
1591
1592                 return ret;
1593         }
1594
1595         if (nandc->buf_start < nandc->buf_count)
1596                 ret = buf[nandc->buf_start++];
1597
1598         return ret;
1599 }
1600
1601 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
1602 {
1603         struct nand_chip *chip = mtd_to_nand(mtd);
1604         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1605         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1606
1607         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
1608         nandc->buf_start += real_len;
1609 }
1610
1611 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
1612                                  int len)
1613 {
1614         struct nand_chip *chip = mtd_to_nand(mtd);
1615         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1616         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1617
1618         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
1619
1620         nandc->buf_start += real_len;
1621 }
1622
1623 /* we support only one external chip for now */
1624 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
1625 {
1626         struct nand_chip *chip = mtd_to_nand(mtd);
1627         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1628
1629         if (chipnr <= 0)
1630                 return;
1631
1632         dev_warn(nandc->dev, "invalid chip select\n");
1633 }
1634
1635 /*
1636  * NAND controller page layout info
1637  *
1638  * Layout with ECC enabled:
1639  *
1640  * |----------------------|  |---------------------------------|
1641  * |           xx.......yy|  |             *********xx.......yy|
1642  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
1643  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
1644  * |           xx.......yy|  |             *********xx.......yy|
1645  * |----------------------|  |---------------------------------|
1646  *     codeword 1,2..n-1                  codeword n
1647  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
1648  *
1649  * n = Number of codewords in the page
1650  * . = ECC bytes
1651  * * = Spare/free bytes
1652  * x = Unused byte(s)
1653  * y = Reserved byte(s)
1654  *
1655  * 2K page: n = 4, spare = 16 bytes
1656  * 4K page: n = 8, spare = 32 bytes
1657  * 8K page: n = 16, spare = 64 bytes
1658  *
1659  * the qcom nand controller operates at a sub page/codeword level. each
1660  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
1661  * the number of ECC bytes vary based on the ECC strength and the bus width.
1662  *
1663  * the first n - 1 codewords contains 516 bytes of user data, the remaining
1664  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
1665  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
1666  *
1667  * When we access a page with ECC enabled, the reserved bytes(s) are not
1668  * accessible at all. When reading, we fill up these unreadable positions
1669  * with 0xffs. When writing, the controller skips writing the inaccessible
1670  * bytes.
1671  *
1672  * Layout with ECC disabled:
1673  *
1674  * |------------------------------|  |---------------------------------------|
1675  * |         yy          xx.......|  |         bb          *********xx.......|
1676  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
1677  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
1678  * |         yy          xx.......|  |         bb          *********xx.......|
1679  * |------------------------------|  |---------------------------------------|
1680  *         codeword 1,2..n-1                        codeword n
1681  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
1682  *
1683  * n = Number of codewords in the page
1684  * . = ECC bytes
1685  * * = Spare/free bytes
1686  * x = Unused byte(s)
1687  * y = Dummy Bad Bock byte(s)
1688  * b = Real Bad Block byte(s)
1689  * size1/size2 = function of codeword size and 'n'
1690  *
1691  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
1692  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
1693  * Block Markers. In the last codeword, this position contains the real BBM
1694  *
1695  * In order to have a consistent layout between RAW and ECC modes, we assume
1696  * the following OOB layout arrangement:
1697  *
1698  * |-----------|  |--------------------|
1699  * |yyxx.......|  |bb*********xx.......|
1700  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
1701  * |yyxx.......|  |bb*********xx.......|
1702  * |yyxx.......|  |bb*********xx.......|
1703  * |-----------|  |--------------------|
1704  *  first n - 1       nth OOB region
1705  *  OOB regions
1706  *
1707  * n = Number of codewords in the page
1708  * . = ECC bytes
1709  * * = FREE OOB bytes
1710  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
1711  * x = Unused byte(s)
1712  * b = Real bad block byte(s) (inaccessible when ECC enabled)
1713  *
1714  * This layout is read as is when ECC is disabled. When ECC is enabled, the
1715  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
1716  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
1717  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
1718  * the sum of the three).
1719  */
1720 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1721                                    struct mtd_oob_region *oobregion)
1722 {
1723         struct nand_chip *chip = mtd_to_nand(mtd);
1724         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1725         struct nand_ecc_ctrl *ecc = &chip->ecc;
1726
1727         if (section > 1)
1728                 return -ERANGE;
1729
1730         if (!section) {
1731                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
1732                                     host->bbm_size;
1733                 oobregion->offset = 0;
1734         } else {
1735                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
1736                 oobregion->offset = mtd->oobsize - oobregion->length;
1737         }
1738
1739         return 0;
1740 }
1741
1742 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
1743                                      struct mtd_oob_region *oobregion)
1744 {
1745         struct nand_chip *chip = mtd_to_nand(mtd);
1746         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1747         struct nand_ecc_ctrl *ecc = &chip->ecc;
1748
1749         if (section)
1750                 return -ERANGE;
1751
1752         oobregion->length = ecc->steps * 4;
1753         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
1754
1755         return 0;
1756 }
1757
1758 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
1759         .ecc = qcom_nand_ooblayout_ecc,
1760         .free = qcom_nand_ooblayout_free,
1761 };
1762
1763 static int qcom_nand_host_setup(struct qcom_nand_host *host)
1764 {
1765         struct nand_chip *chip = &host->chip;
1766         struct mtd_info *mtd = nand_to_mtd(chip);
1767         struct nand_ecc_ctrl *ecc = &chip->ecc;
1768         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1769         int cwperpage, bad_block_byte;
1770         bool wide_bus;
1771         int ecc_mode = 1;
1772
1773         /*
1774          * the controller requires each step consists of 512 bytes of data.
1775          * bail out if DT has populated a wrong step size.
1776          */
1777         if (ecc->size != NANDC_STEP_SIZE) {
1778                 dev_err(nandc->dev, "invalid ecc size\n");
1779                 return -EINVAL;
1780         }
1781
1782         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
1783
1784         if (ecc->strength >= 8) {
1785                 /* 8 bit ECC defaults to BCH ECC on all platforms */
1786                 host->bch_enabled = true;
1787                 ecc_mode = 1;
1788
1789                 if (wide_bus) {
1790                         host->ecc_bytes_hw = 14;
1791                         host->spare_bytes = 0;
1792                         host->bbm_size = 2;
1793                 } else {
1794                         host->ecc_bytes_hw = 13;
1795                         host->spare_bytes = 2;
1796                         host->bbm_size = 1;
1797                 }
1798         } else {
1799                 /*
1800                  * if the controller supports BCH for 4 bit ECC, the controller
1801                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
1802                  * always 10 bytes
1803                  */
1804                 if (nandc->ecc_modes & ECC_BCH_4BIT) {
1805                         /* BCH */
1806                         host->bch_enabled = true;
1807                         ecc_mode = 0;
1808
1809                         if (wide_bus) {
1810                                 host->ecc_bytes_hw = 8;
1811                                 host->spare_bytes = 2;
1812                                 host->bbm_size = 2;
1813                         } else {
1814                                 host->ecc_bytes_hw = 7;
1815                                 host->spare_bytes = 4;
1816                                 host->bbm_size = 1;
1817                         }
1818                 } else {
1819                         /* RS */
1820                         host->ecc_bytes_hw = 10;
1821
1822                         if (wide_bus) {
1823                                 host->spare_bytes = 0;
1824                                 host->bbm_size = 2;
1825                         } else {
1826                                 host->spare_bytes = 1;
1827                                 host->bbm_size = 1;
1828                         }
1829                 }
1830         }
1831
1832         /*
1833          * we consider ecc->bytes as the sum of all the non-data content in a
1834          * step. It gives us a clean representation of the oob area (even if
1835          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
1836          * ECC and 12 bytes for 4 bit ECC
1837          */
1838         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
1839
1840         ecc->read_page          = qcom_nandc_read_page;
1841         ecc->read_page_raw      = qcom_nandc_read_page_raw;
1842         ecc->read_oob           = qcom_nandc_read_oob;
1843         ecc->write_page         = qcom_nandc_write_page;
1844         ecc->write_page_raw     = qcom_nandc_write_page_raw;
1845         ecc->write_oob          = qcom_nandc_write_oob;
1846
1847         ecc->mode = NAND_ECC_HW;
1848
1849         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
1850
1851         cwperpage = mtd->writesize / ecc->size;
1852
1853         /*
1854          * DATA_UD_BYTES varies based on whether the read/write command protects
1855          * spare data with ECC too. We protect spare data by default, so we set
1856          * it to main + spare data, which are 512 and 4 bytes respectively.
1857          */
1858         host->cw_data = 516;
1859
1860         /*
1861          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
1862          * for 8 bit ECC
1863          */
1864         host->cw_size = host->cw_data + ecc->bytes;
1865
1866         if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
1867                 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
1868                 return -EINVAL;
1869         }
1870
1871         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
1872
1873         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
1874                                 | host->cw_data << UD_SIZE_BYTES
1875                                 | 0 << DISABLE_STATUS_AFTER_WRITE
1876                                 | 5 << NUM_ADDR_CYCLES
1877                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
1878                                 | 0 << STATUS_BFR_READ
1879                                 | 1 << SET_RD_MODE_AFTER_STATUS
1880                                 | host->spare_bytes << SPARE_SIZE_BYTES;
1881
1882         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
1883                                 | 0 <<  CS_ACTIVE_BSY
1884                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
1885                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
1886                                 | 2 << WR_RD_BSY_GAP
1887                                 | wide_bus << WIDE_FLASH
1888                                 | host->bch_enabled << ENABLE_BCH_ECC;
1889
1890         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
1891                                 | host->cw_size << UD_SIZE_BYTES
1892                                 | 5 << NUM_ADDR_CYCLES
1893                                 | 0 << SPARE_SIZE_BYTES;
1894
1895         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
1896                                 | 0 << CS_ACTIVE_BSY
1897                                 | 17 << BAD_BLOCK_BYTE_NUM
1898                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
1899                                 | 2 << WR_RD_BSY_GAP
1900                                 | wide_bus << WIDE_FLASH
1901                                 | 1 << DEV0_CFG1_ECC_DISABLE;
1902
1903         host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
1904                                 | 0 << ECC_SW_RESET
1905                                 | host->cw_data << ECC_NUM_DATA_BYTES
1906                                 | 1 << ECC_FORCE_CLK_OPEN
1907                                 | ecc_mode << ECC_MODE
1908                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
1909
1910         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
1911
1912         host->clrflashstatus = FS_READY_BSY_N;
1913         host->clrreadstatus = 0xc0;
1914
1915         dev_dbg(nandc->dev,
1916                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
1917                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
1918                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
1919                 cwperpage);
1920
1921         return 0;
1922 }
1923
1924 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
1925 {
1926         int ret;
1927
1928         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
1929         if (ret) {
1930                 dev_err(nandc->dev, "failed to set DMA mask\n");
1931                 return ret;
1932         }
1933
1934         /*
1935          * we use the internal buffer for reading ONFI params, reading small
1936          * data like ID and status, and preforming read-copy-write operations
1937          * when writing to a codeword partially. 532 is the maximum possible
1938          * size of a codeword for our nand controller
1939          */
1940         nandc->buf_size = 532;
1941
1942         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
1943                                         GFP_KERNEL);
1944         if (!nandc->data_buffer)
1945                 return -ENOMEM;
1946
1947         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
1948                                         GFP_KERNEL);
1949         if (!nandc->regs)
1950                 return -ENOMEM;
1951
1952         nandc->reg_read_buf = devm_kzalloc(nandc->dev,
1953                                 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
1954                                 GFP_KERNEL);
1955         if (!nandc->reg_read_buf)
1956                 return -ENOMEM;
1957
1958         nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
1959         if (!nandc->chan) {
1960                 dev_err(nandc->dev, "failed to request slave channel\n");
1961                 return -ENODEV;
1962         }
1963
1964         INIT_LIST_HEAD(&nandc->desc_list);
1965         INIT_LIST_HEAD(&nandc->host_list);
1966
1967         nand_hw_control_init(&nandc->controller);
1968
1969         return 0;
1970 }
1971
1972 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
1973 {
1974         dma_release_channel(nandc->chan);
1975 }
1976
1977 /* one time setup of a few nand controller registers */
1978 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
1979 {
1980         /* kill onenand */
1981         nandc_write(nandc, SFLASHC_BURST_CFG, 0);
1982         nandc_write(nandc, NAND_DEV_CMD_VLD, NAND_DEV_CMD_VLD_VAL);
1983
1984         /* enable ADM DMA */
1985         nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
1986
1987         /* save the original values of these registers */
1988         nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
1989         nandc->vld = NAND_DEV_CMD_VLD_VAL;
1990
1991         return 0;
1992 }
1993
1994 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
1995                                struct qcom_nand_host *host,
1996                                struct device_node *dn)
1997 {
1998         struct nand_chip *chip = &host->chip;
1999         struct mtd_info *mtd = nand_to_mtd(chip);
2000         struct device *dev = nandc->dev;
2001         int ret;
2002
2003         ret = of_property_read_u32(dn, "reg", &host->cs);
2004         if (ret) {
2005                 dev_err(dev, "can't get chip-select\n");
2006                 return -ENXIO;
2007         }
2008
2009         nand_set_flash_node(chip, dn);
2010         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2011         if (!mtd->name)
2012                 return -ENOMEM;
2013
2014         mtd->owner = THIS_MODULE;
2015         mtd->dev.parent = dev;
2016
2017         chip->cmdfunc           = qcom_nandc_command;
2018         chip->select_chip       = qcom_nandc_select_chip;
2019         chip->read_byte         = qcom_nandc_read_byte;
2020         chip->read_buf          = qcom_nandc_read_buf;
2021         chip->write_buf         = qcom_nandc_write_buf;
2022
2023         /*
2024          * the bad block marker is readable only when we read the last codeword
2025          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2026          * helpers don't allow us to read BB from a nand chip with ECC
2027          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2028          * and block_markbad helpers until we permanently switch to using
2029          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2030          */
2031         chip->block_bad         = qcom_nandc_block_bad;
2032         chip->block_markbad     = qcom_nandc_block_markbad;
2033
2034         chip->controller = &nandc->controller;
2035         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2036                          NAND_SKIP_BBTSCAN;
2037
2038         /* set up initial status value */
2039         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2040
2041         ret = nand_scan_ident(mtd, 1, NULL);
2042         if (ret)
2043                 return ret;
2044
2045         ret = qcom_nand_host_setup(host);
2046         if (ret)
2047                 return ret;
2048
2049         ret = nand_scan_tail(mtd);
2050         if (ret)
2051                 return ret;
2052
2053         return mtd_device_register(mtd, NULL, 0);
2054 }
2055
2056 /* parse custom DT properties here */
2057 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2058 {
2059         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2060         struct device_node *np = nandc->dev->of_node;
2061         int ret;
2062
2063         ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
2064         if (ret) {
2065                 dev_err(nandc->dev, "command CRCI unspecified\n");
2066                 return ret;
2067         }
2068
2069         ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
2070         if (ret) {
2071                 dev_err(nandc->dev, "data CRCI unspecified\n");
2072                 return ret;
2073         }
2074
2075         return 0;
2076 }
2077
2078 static int qcom_nandc_probe(struct platform_device *pdev)
2079 {
2080         struct qcom_nand_controller *nandc;
2081         struct qcom_nand_host *host;
2082         const void *dev_data;
2083         struct device *dev = &pdev->dev;
2084         struct device_node *dn = dev->of_node, *child;
2085         struct resource *res;
2086         int ret;
2087
2088         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2089         if (!nandc)
2090                 return -ENOMEM;
2091
2092         platform_set_drvdata(pdev, nandc);
2093         nandc->dev = dev;
2094
2095         dev_data = of_device_get_match_data(dev);
2096         if (!dev_data) {
2097                 dev_err(&pdev->dev, "failed to get device data\n");
2098                 return -ENODEV;
2099         }
2100
2101         nandc->ecc_modes = (unsigned long)dev_data;
2102
2103         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2104         nandc->base = devm_ioremap_resource(dev, res);
2105         if (IS_ERR(nandc->base))
2106                 return PTR_ERR(nandc->base);
2107
2108         nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2109
2110         nandc->core_clk = devm_clk_get(dev, "core");
2111         if (IS_ERR(nandc->core_clk))
2112                 return PTR_ERR(nandc->core_clk);
2113
2114         nandc->aon_clk = devm_clk_get(dev, "aon");
2115         if (IS_ERR(nandc->aon_clk))
2116                 return PTR_ERR(nandc->aon_clk);
2117
2118         ret = qcom_nandc_parse_dt(pdev);
2119         if (ret)
2120                 return ret;
2121
2122         ret = qcom_nandc_alloc(nandc);
2123         if (ret)
2124                 return ret;
2125
2126         ret = clk_prepare_enable(nandc->core_clk);
2127         if (ret)
2128                 goto err_core_clk;
2129
2130         ret = clk_prepare_enable(nandc->aon_clk);
2131         if (ret)
2132                 goto err_aon_clk;
2133
2134         ret = qcom_nandc_setup(nandc);
2135         if (ret)
2136                 goto err_setup;
2137
2138         for_each_available_child_of_node(dn, child) {
2139                 if (of_device_is_compatible(child, "qcom,nandcs")) {
2140                         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2141                         if (!host) {
2142                                 of_node_put(child);
2143                                 ret = -ENOMEM;
2144                                 goto err_cs_init;
2145                         }
2146
2147                         ret = qcom_nand_host_init(nandc, host, child);
2148                         if (ret) {
2149                                 devm_kfree(dev, host);
2150                                 continue;
2151                         }
2152
2153                         list_add_tail(&host->node, &nandc->host_list);
2154                 }
2155         }
2156
2157         if (list_empty(&nandc->host_list)) {
2158                 ret = -ENODEV;
2159                 goto err_cs_init;
2160         }
2161
2162         return 0;
2163
2164 err_cs_init:
2165         list_for_each_entry(host, &nandc->host_list, node)
2166                 nand_release(&host->chip);
2167 err_setup:
2168         clk_disable_unprepare(nandc->aon_clk);
2169 err_aon_clk:
2170         clk_disable_unprepare(nandc->core_clk);
2171 err_core_clk:
2172         qcom_nandc_unalloc(nandc);
2173
2174         return ret;
2175 }
2176
2177 static int qcom_nandc_remove(struct platform_device *pdev)
2178 {
2179         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2180         struct qcom_nand_host *host;
2181
2182         list_for_each_entry(host, &nandc->host_list, node)
2183                 nand_release(&host->chip);
2184
2185         qcom_nandc_unalloc(nandc);
2186
2187         clk_disable_unprepare(nandc->aon_clk);
2188         clk_disable_unprepare(nandc->core_clk);
2189
2190         return 0;
2191 }
2192
2193 #define EBI2_NANDC_ECC_MODES    (ECC_RS_4BIT | ECC_BCH_8BIT)
2194
2195 /*
2196  * data will hold a struct pointer containing more differences once we support
2197  * more controller variants
2198  */
2199 static const struct of_device_id qcom_nandc_of_match[] = {
2200         {       .compatible = "qcom,ipq806x-nand",
2201                 .data = (void *)EBI2_NANDC_ECC_MODES,
2202         },
2203         {}
2204 };
2205 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2206
2207 static struct platform_driver qcom_nandc_driver = {
2208         .driver = {
2209                 .name = "qcom-nandc",
2210                 .of_match_table = qcom_nandc_of_match,
2211         },
2212         .probe   = qcom_nandc_probe,
2213         .remove  = qcom_nandc_remove,
2214 };
2215 module_platform_driver(qcom_nandc_driver);
2216
2217 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2218 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2219 MODULE_LICENSE("GPL v2");