2 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
14 #include <linux/clk.h>
15 #include <linux/slab.h>
16 #include <linux/bitops.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmaengine.h>
19 #include <linux/module.h>
20 #include <linux/mtd/rawnand.h>
21 #include <linux/mtd/partitions.h>
23 #include <linux/of_device.h>
24 #include <linux/delay.h>
26 /* NANDc reg offsets */
27 #define NAND_FLASH_CMD 0x00
28 #define NAND_ADDR0 0x04
29 #define NAND_ADDR1 0x08
30 #define NAND_FLASH_CHIP_SELECT 0x0c
31 #define NAND_EXEC_CMD 0x10
32 #define NAND_FLASH_STATUS 0x14
33 #define NAND_BUFFER_STATUS 0x18
34 #define NAND_DEV0_CFG0 0x20
35 #define NAND_DEV0_CFG1 0x24
36 #define NAND_DEV0_ECC_CFG 0x28
37 #define NAND_DEV1_ECC_CFG 0x2c
38 #define NAND_DEV1_CFG0 0x30
39 #define NAND_DEV1_CFG1 0x34
40 #define NAND_READ_ID 0x40
41 #define NAND_READ_STATUS 0x44
42 #define NAND_DEV_CMD0 0xa0
43 #define NAND_DEV_CMD1 0xa4
44 #define NAND_DEV_CMD2 0xa8
45 #define NAND_DEV_CMD_VLD 0xac
46 #define SFLASHC_BURST_CFG 0xe0
47 #define NAND_ERASED_CW_DETECT_CFG 0xe8
48 #define NAND_ERASED_CW_DETECT_STATUS 0xec
49 #define NAND_EBI2_ECC_BUF_CFG 0xf0
50 #define FLASH_BUF_ACC 0x100
52 #define NAND_CTRL 0xf00
53 #define NAND_VERSION 0xf08
54 #define NAND_READ_LOCATION_0 0xf20
55 #define NAND_READ_LOCATION_1 0xf24
56 #define NAND_READ_LOCATION_2 0xf28
57 #define NAND_READ_LOCATION_3 0xf2c
59 /* dummy register offsets, used by write_reg_dma */
60 #define NAND_DEV_CMD1_RESTORE 0xdead
61 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef
63 /* NAND_FLASH_CMD bits */
64 #define PAGE_ACC BIT(4)
65 #define LAST_PAGE BIT(5)
67 /* NAND_FLASH_CHIP_SELECT bits */
68 #define NAND_DEV_SEL 0
71 /* NAND_FLASH_STATUS bits */
72 #define FS_OP_ERR BIT(4)
73 #define FS_READY_BSY_N BIT(5)
74 #define FS_MPU_ERR BIT(8)
75 #define FS_DEVICE_STS_ERR BIT(16)
76 #define FS_DEVICE_WP BIT(23)
78 /* NAND_BUFFER_STATUS bits */
79 #define BS_UNCORRECTABLE_BIT BIT(8)
80 #define BS_CORRECTABLE_ERR_MSK 0x1f
82 /* NAND_DEVn_CFG0 bits */
83 #define DISABLE_STATUS_AFTER_WRITE 4
85 #define UD_SIZE_BYTES 9
86 #define ECC_PARITY_SIZE_BYTES_RS 19
87 #define SPARE_SIZE_BYTES 23
88 #define NUM_ADDR_CYCLES 27
89 #define STATUS_BFR_READ 30
90 #define SET_RD_MODE_AFTER_STATUS 31
92 /* NAND_DEVn_CFG0 bits */
93 #define DEV0_CFG1_ECC_DISABLE 0
95 #define NAND_RECOVERY_CYCLES 2
96 #define CS_ACTIVE_BSY 5
97 #define BAD_BLOCK_BYTE_NUM 6
98 #define BAD_BLOCK_IN_SPARE_AREA 16
99 #define WR_RD_BSY_GAP 17
100 #define ENABLE_BCH_ECC 27
102 /* NAND_DEV0_ECC_CFG bits */
103 #define ECC_CFG_ECC_DISABLE 0
104 #define ECC_SW_RESET 1
106 #define ECC_PARITY_SIZE_BYTES_BCH 8
107 #define ECC_NUM_DATA_BYTES 16
108 #define ECC_FORCE_CLK_OPEN 30
110 /* NAND_DEV_CMD1 bits */
113 /* NAND_DEV_CMD_VLD bits */
114 #define READ_START_VLD BIT(0)
115 #define READ_STOP_VLD BIT(1)
116 #define WRITE_START_VLD BIT(2)
117 #define ERASE_START_VLD BIT(3)
118 #define SEQ_READ_START_VLD BIT(4)
120 /* NAND_EBI2_ECC_BUF_CFG bits */
123 /* NAND_ERASED_CW_DETECT_CFG bits */
124 #define ERASED_CW_ECC_MASK 1
125 #define AUTO_DETECT_RES 0
126 #define MASK_ECC (1 << ERASED_CW_ECC_MASK)
127 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
128 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
129 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
130 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
132 /* NAND_ERASED_CW_DETECT_STATUS bits */
133 #define PAGE_ALL_ERASED BIT(7)
134 #define CODEWORD_ALL_ERASED BIT(6)
135 #define PAGE_ERASED BIT(5)
136 #define CODEWORD_ERASED BIT(4)
137 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
138 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
140 /* NAND_READ_LOCATION_n bits */
141 #define READ_LOCATION_OFFSET 0
142 #define READ_LOCATION_SIZE 16
143 #define READ_LOCATION_LAST 31
146 #define NAND_VERSION_MAJOR_MASK 0xf0000000
147 #define NAND_VERSION_MAJOR_SHIFT 28
148 #define NAND_VERSION_MINOR_MASK 0x0fff0000
149 #define NAND_VERSION_MINOR_SHIFT 16
152 #define OP_PAGE_READ 0x2
153 #define OP_PAGE_READ_WITH_ECC 0x3
154 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4
155 #define OP_PROGRAM_PAGE 0x6
156 #define OP_PAGE_PROGRAM_WITH_ECC 0x7
157 #define OP_PROGRAM_PAGE_SPARE 0x9
158 #define OP_BLOCK_ERASE 0xa
159 #define OP_FETCH_ID 0xb
160 #define OP_RESET_DEVICE 0xd
162 /* Default Value for NAND_DEV_CMD_VLD */
163 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \
164 ERASE_START_VLD | SEQ_READ_START_VLD)
167 #define BAM_MODE_EN BIT(0)
170 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
171 * the driver calls the chunks 'step' or 'codeword' interchangeably
173 #define NANDC_STEP_SIZE 512
176 * the largest page size we support is 8K, this will have 16 steps/codewords
179 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
181 /* we read at most 3 registers per codeword scan */
182 #define MAX_REG_RD (3 * MAX_NUM_STEPS)
184 /* ECC modes supported by the controller */
185 #define ECC_NONE BIT(0)
186 #define ECC_RS_4BIT BIT(1)
187 #define ECC_BCH_4BIT BIT(2)
188 #define ECC_BCH_8BIT BIT(3)
190 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \
191 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \
192 ((offset) << READ_LOCATION_OFFSET) | \
193 ((size) << READ_LOCATION_SIZE) | \
194 ((is_last) << READ_LOCATION_LAST))
197 * Returns the actual register address for all NAND_DEV_ registers
198 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
200 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
202 #define QPIC_PER_CW_CMD_SGL 32
203 #define QPIC_PER_CW_DATA_SGL 8
206 * Flags used in DMA descriptor preparation helper functions
207 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
209 /* Don't set the EOT in current tx BAM sgl */
210 #define NAND_BAM_NO_EOT BIT(0)
211 /* Set the NWD flag in current BAM sgl */
212 #define NAND_BAM_NWD BIT(1)
213 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
214 #define NAND_BAM_NEXT_SGL BIT(2)
216 * Erased codeword status is being used two times in single transfer so this
217 * flag will determine the current value of erased codeword status register
219 #define NAND_ERASED_CW_SET BIT(4)
222 * This data type corresponds to the BAM transaction which will be used for all
224 * @cmd_sgl - sgl for NAND BAM command pipe
225 * @data_sgl - sgl for NAND BAM consumer/producer pipe
226 * @cmd_sgl_pos - current index in command sgl.
227 * @cmd_sgl_start - start index in command sgl.
228 * @tx_sgl_pos - current index in data sgl for tx.
229 * @tx_sgl_start - start index in data sgl for tx.
230 * @rx_sgl_pos - current index in data sgl for rx.
231 * @rx_sgl_start - start index in data sgl for rx.
233 struct bam_transaction {
234 struct scatterlist *cmd_sgl;
235 struct scatterlist *data_sgl;
245 * This data type corresponds to the nand dma descriptor
246 * @list - list for desc_info
247 * @dir - DMA transfer direction
248 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
250 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
251 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
252 * @dma_desc - low level DMA engine descriptor
255 struct list_head node;
257 enum dma_data_direction dir;
259 struct scatterlist adm_sgl;
261 struct scatterlist *bam_sgl;
265 struct dma_async_tx_descriptor *dma_desc;
269 * holds the current register values that we want to write. acts as a contiguous
270 * chunk of memory which we use to write the controller registers through DMA.
283 __le32 clrflashstatus;
284 __le32 clrreadstatus;
293 __le32 read_location0;
294 __le32 read_location1;
295 __le32 read_location2;
296 __le32 read_location3;
298 __le32 erased_cw_detect_cfg_clr;
299 __le32 erased_cw_detect_cfg_set;
303 * NAND controller data struct
305 * @controller: base controller structure
306 * @host_list: list containing all the chips attached to the
308 * @dev: parent device
310 * @base_dma: physical base address of controller registers
311 * @core_clk: controller clock
312 * @aon_clk: another controller clock
315 * @cmd_crci: ADM DMA CRCI for command flow control
316 * @data_crci: ADM DMA CRCI for data flow control
317 * @desc_list: DMA descriptor list (list of desc_infos)
319 * @data_buffer: our local DMA buffer for page read/writes,
320 * used when we can't use the buffer provided
321 * by upper layers directly
322 * @buf_size/count/start: markers for chip->read_buf/write_buf functions
323 * @reg_read_buf: local buffer for reading back registers via DMA
324 * @reg_read_dma: contains dma address for register read buffer
325 * @reg_read_pos: marker for data read in reg_read_buf
327 * @regs: a contiguous chunk of memory for DMA register
328 * writes. contains the register values to be
329 * written to controller
330 * @cmd1/vld: some fixed controller register values
331 * @props: properties of current NAND controller,
332 * initialized via DT match data
333 * @max_cwperpage: maximum QPIC codewords required. calculated
334 * from all connected NAND devices pagesize
336 struct qcom_nand_controller {
337 struct nand_hw_control controller;
338 struct list_head host_list;
345 struct clk *core_clk;
349 /* will be used only by QPIC for BAM DMA */
351 struct dma_chan *tx_chan;
352 struct dma_chan *rx_chan;
353 struct dma_chan *cmd_chan;
356 /* will be used only by EBI2 for ADM DMA */
358 struct dma_chan *chan;
359 unsigned int cmd_crci;
360 unsigned int data_crci;
364 struct list_head desc_list;
365 struct bam_transaction *bam_txn;
371 unsigned int max_cwperpage;
373 __le32 *reg_read_buf;
374 dma_addr_t reg_read_dma;
377 struct nandc_regs *regs;
380 const struct qcom_nandc_props *props;
384 * NAND chip structure
386 * @chip: base NAND chip structure
387 * @node: list node to add itself to host_list in
388 * qcom_nand_controller
390 * @cs: chip select value for this chip
391 * @cw_size: the number of bytes in a single step/codeword
392 * of a page, consisting of all data, ecc, spare
394 * @cw_data: the number of bytes within a codeword protected
396 * @use_ecc: request the controller to use ECC for the
397 * upcoming read/write
398 * @bch_enabled: flag to tell whether BCH ECC mode is used
399 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
401 * @status: value to be returned if NAND_CMD_STATUS command
403 * @last_command: keeps track of last command on this chip. used
404 * for reading correct status
406 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
407 * ecc/non-ecc mode for the current nand flash
410 struct qcom_nand_host {
411 struct nand_chip chip;
412 struct list_head node;
426 u32 cfg0_raw, cfg1_raw;
434 * This data type corresponds to the NAND controller properties which varies
435 * among different NAND controllers.
436 * @ecc_modes - ecc mode for NAND
437 * @is_bam - whether NAND controller is using BAM
438 * @is_qpic - whether NAND CTRL is part of qpic IP
439 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
441 struct qcom_nandc_props {
445 u32 dev_cmd_reg_start;
448 /* Frees the BAM transaction memory */
449 static void free_bam_transaction(struct qcom_nand_controller *nandc)
451 struct bam_transaction *bam_txn = nandc->bam_txn;
453 devm_kfree(nandc->dev, bam_txn);
456 /* Allocates and Initializes the BAM transaction */
457 static struct bam_transaction *
458 alloc_bam_transaction(struct qcom_nand_controller *nandc)
460 struct bam_transaction *bam_txn;
462 unsigned int num_cw = nandc->max_cwperpage;
466 sizeof(*bam_txn) + num_cw *
467 ((sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
468 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
470 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
474 bam_txn = bam_txn_buf;
475 bam_txn_buf += sizeof(*bam_txn);
477 bam_txn->cmd_sgl = bam_txn_buf;
479 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
481 bam_txn->data_sgl = bam_txn_buf;
486 /* Clears the BAM transaction indexes */
487 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
489 struct bam_transaction *bam_txn = nandc->bam_txn;
491 if (!nandc->props->is_bam)
494 bam_txn->cmd_sgl_pos = 0;
495 bam_txn->cmd_sgl_start = 0;
496 bam_txn->tx_sgl_pos = 0;
497 bam_txn->tx_sgl_start = 0;
498 bam_txn->rx_sgl_pos = 0;
499 bam_txn->rx_sgl_start = 0;
501 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
502 QPIC_PER_CW_CMD_SGL);
503 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
504 QPIC_PER_CW_DATA_SGL);
507 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
509 return container_of(chip, struct qcom_nand_host, chip);
512 static inline struct qcom_nand_controller *
513 get_qcom_nand_controller(struct nand_chip *chip)
515 return container_of(chip->controller, struct qcom_nand_controller,
519 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
521 return ioread32(nandc->base + offset);
524 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
527 iowrite32(val, nandc->base + offset);
530 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
533 if (!nandc->props->is_bam)
537 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
539 sizeof(*nandc->reg_read_buf),
542 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
544 sizeof(*nandc->reg_read_buf),
548 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
557 case NAND_FLASH_CHIP_SELECT:
558 return ®s->chip_sel;
561 case NAND_FLASH_STATUS:
562 return ®s->clrflashstatus;
567 case NAND_DEV0_ECC_CFG:
568 return ®s->ecc_bch_cfg;
569 case NAND_READ_STATUS:
570 return ®s->clrreadstatus;
573 case NAND_DEV_CMD1_RESTORE:
574 return ®s->orig_cmd1;
575 case NAND_DEV_CMD_VLD:
577 case NAND_DEV_CMD_VLD_RESTORE:
578 return ®s->orig_vld;
579 case NAND_EBI2_ECC_BUF_CFG:
580 return ®s->ecc_buf_cfg;
581 case NAND_READ_LOCATION_0:
582 return ®s->read_location0;
583 case NAND_READ_LOCATION_1:
584 return ®s->read_location1;
585 case NAND_READ_LOCATION_2:
586 return ®s->read_location2;
587 case NAND_READ_LOCATION_3:
588 return ®s->read_location3;
594 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
597 struct nandc_regs *regs = nandc->regs;
600 reg = offset_to_nandc_reg(regs, offset);
603 *reg = cpu_to_le32(val);
606 /* helper to configure address register values */
607 static void set_address(struct qcom_nand_host *host, u16 column, int page)
609 struct nand_chip *chip = &host->chip;
610 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
612 if (chip->options & NAND_BUSWIDTH_16)
615 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
616 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
620 * update_rw_regs: set up read/write register values, these will be
621 * written to the NAND controller registers via DMA
623 * @num_cw: number of steps for the read/write operation
624 * @read: read or write operation
626 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
628 struct nand_chip *chip = &host->chip;
629 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
630 u32 cmd, cfg0, cfg1, ecc_bch_cfg;
634 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
636 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
638 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
642 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
643 (num_cw - 1) << CW_PER_PAGE;
646 ecc_bch_cfg = host->ecc_bch_cfg;
648 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
649 (num_cw - 1) << CW_PER_PAGE;
651 cfg1 = host->cfg1_raw;
652 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
655 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
656 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
657 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
658 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
659 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
660 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
661 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
662 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
665 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
666 host->cw_data : host->cw_size, 1);
670 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
671 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
672 * which will be submitted to DMA engine.
674 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
675 struct dma_chan *chan,
678 struct desc_info *desc;
679 struct scatterlist *sgl;
680 unsigned int sgl_cnt;
682 struct bam_transaction *bam_txn = nandc->bam_txn;
683 enum dma_transfer_direction dir_eng;
684 struct dma_async_tx_descriptor *dma_desc;
686 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
690 if (chan == nandc->cmd_chan) {
691 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
692 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
693 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
694 dir_eng = DMA_MEM_TO_DEV;
695 desc->dir = DMA_TO_DEVICE;
696 } else if (chan == nandc->tx_chan) {
697 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
698 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
699 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
700 dir_eng = DMA_MEM_TO_DEV;
701 desc->dir = DMA_TO_DEVICE;
703 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
704 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
705 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
706 dir_eng = DMA_DEV_TO_MEM;
707 desc->dir = DMA_FROM_DEVICE;
710 sg_mark_end(sgl + sgl_cnt - 1);
711 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
713 dev_err(nandc->dev, "failure in mapping desc\n");
718 desc->sgl_cnt = sgl_cnt;
721 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
725 dev_err(nandc->dev, "failure in prep desc\n");
726 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
731 desc->dma_desc = dma_desc;
733 list_add_tail(&desc->node, &nandc->desc_list);
739 * Prepares the data descriptor for BAM DMA which will be used for NAND
740 * data reads and writes.
742 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
744 int size, unsigned int flags)
747 struct bam_transaction *bam_txn = nandc->bam_txn;
750 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
752 bam_txn->rx_sgl_pos++;
754 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
756 bam_txn->tx_sgl_pos++;
759 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
760 * is not set, form the DMA descriptor
762 if (!(flags & NAND_BAM_NO_EOT)) {
763 ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
773 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
774 int reg_off, const void *vaddr, int size,
777 struct desc_info *desc;
778 struct dma_async_tx_descriptor *dma_desc;
779 struct scatterlist *sgl;
780 struct dma_slave_config slave_conf;
781 enum dma_transfer_direction dir_eng;
784 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
788 sgl = &desc->adm_sgl;
790 sg_init_one(sgl, vaddr, size);
793 dir_eng = DMA_DEV_TO_MEM;
794 desc->dir = DMA_FROM_DEVICE;
796 dir_eng = DMA_MEM_TO_DEV;
797 desc->dir = DMA_TO_DEVICE;
800 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
806 memset(&slave_conf, 0x00, sizeof(slave_conf));
808 slave_conf.device_fc = flow_control;
810 slave_conf.src_maxburst = 16;
811 slave_conf.src_addr = nandc->base_dma + reg_off;
812 slave_conf.slave_id = nandc->data_crci;
814 slave_conf.dst_maxburst = 16;
815 slave_conf.dst_addr = nandc->base_dma + reg_off;
816 slave_conf.slave_id = nandc->cmd_crci;
819 ret = dmaengine_slave_config(nandc->chan, &slave_conf);
821 dev_err(nandc->dev, "failed to configure dma channel\n");
825 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
827 dev_err(nandc->dev, "failed to prepare desc\n");
832 desc->dma_desc = dma_desc;
834 list_add_tail(&desc->node, &nandc->desc_list);
844 * read_reg_dma: prepares a descriptor to read a given number of
845 * contiguous registers to the reg_read_buf pointer
847 * @first: offset of the first register in the contiguous block
848 * @num_regs: number of registers to read
849 * @flags: flags to control DMA descriptor preparation
851 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
852 int num_regs, unsigned int flags)
854 bool flow_control = false;
858 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
861 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
862 first = dev_cmd_reg_addr(nandc, first);
864 size = num_regs * sizeof(u32);
865 vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
866 nandc->reg_read_pos += num_regs;
868 return prep_adm_dma_desc(nandc, true, first, vaddr, size, flow_control);
872 * write_reg_dma: prepares a descriptor to write a given number of
873 * contiguous registers
875 * @first: offset of the first register in the contiguous block
876 * @num_regs: number of registers to write
877 * @flags: flags to control DMA descriptor preparation
879 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
880 int num_regs, unsigned int flags)
882 bool flow_control = false;
883 struct nandc_regs *regs = nandc->regs;
887 vaddr = offset_to_nandc_reg(regs, first);
889 if (first == NAND_FLASH_CMD)
892 if (first == NAND_ERASED_CW_DETECT_CFG) {
893 if (flags & NAND_ERASED_CW_SET)
894 vaddr = ®s->erased_cw_detect_cfg_set;
896 vaddr = ®s->erased_cw_detect_cfg_clr;
899 if (first == NAND_EXEC_CMD)
900 flags |= NAND_BAM_NWD;
902 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
903 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
905 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
906 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
908 size = num_regs * sizeof(u32);
910 return prep_adm_dma_desc(nandc, false, first, vaddr, size,
915 * read_data_dma: prepares a DMA descriptor to transfer data from the
916 * controller's internal buffer to the buffer 'vaddr'
918 * @reg_off: offset within the controller's data buffer
919 * @vaddr: virtual address of the buffer we want to write to
920 * @size: DMA transaction size in bytes
921 * @flags: flags to control DMA descriptor preparation
923 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
924 const u8 *vaddr, int size, unsigned int flags)
926 if (nandc->props->is_bam)
927 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
929 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
933 * write_data_dma: prepares a DMA descriptor to transfer data from
934 * 'vaddr' to the controller's internal buffer
936 * @reg_off: offset within the controller's data buffer
937 * @vaddr: virtual address of the buffer we want to read from
938 * @size: DMA transaction size in bytes
939 * @flags: flags to control DMA descriptor preparation
941 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
942 const u8 *vaddr, int size, unsigned int flags)
944 if (nandc->props->is_bam)
945 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
947 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
951 * Helper to prepare DMA descriptors for configuring registers
952 * before reading a NAND page.
954 static void config_nand_page_read(struct qcom_nand_controller *nandc)
956 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
957 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
958 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
959 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
960 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
961 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
965 * Helper to prepare DMA descriptors for configuring registers
966 * before reading each codeword in NAND page.
968 static void config_nand_cw_read(struct qcom_nand_controller *nandc)
970 if (nandc->props->is_bam)
971 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
974 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
975 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
977 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
978 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
983 * Helper to prepare dma descriptors to configure registers needed for reading a
984 * single codeword in page
986 static void config_nand_single_cw_page_read(struct qcom_nand_controller *nandc)
988 config_nand_page_read(nandc);
989 config_nand_cw_read(nandc);
993 * Helper to prepare DMA descriptors used to configure registers needed for
994 * before writing a NAND page.
996 static void config_nand_page_write(struct qcom_nand_controller *nandc)
998 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
999 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1000 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1005 * Helper to prepare DMA descriptors for configuring registers
1006 * before writing each codeword in NAND page.
1008 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1010 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1011 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1013 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1015 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1016 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1020 * the following functions are used within chip->cmdfunc() to perform different
1021 * NAND_CMD_* commands
1024 /* sets up descriptors for NAND_CMD_PARAM */
1025 static int nandc_param(struct qcom_nand_host *host)
1027 struct nand_chip *chip = &host->chip;
1028 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1031 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1032 * in use. we configure the controller to perform a raw read of 512
1033 * bytes to read onfi params
1035 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1036 nandc_set_reg(nandc, NAND_ADDR0, 0);
1037 nandc_set_reg(nandc, NAND_ADDR1, 0);
1038 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1039 | 512 << UD_SIZE_BYTES
1040 | 5 << NUM_ADDR_CYCLES
1041 | 0 << SPARE_SIZE_BYTES);
1042 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1043 | 0 << CS_ACTIVE_BSY
1044 | 17 << BAD_BLOCK_BYTE_NUM
1045 | 1 << BAD_BLOCK_IN_SPARE_AREA
1046 | 2 << WR_RD_BSY_GAP
1048 | 1 << DEV0_CFG1_ECC_DISABLE);
1049 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1051 /* configure CMD1 and VLD for ONFI param probing */
1052 nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1053 (nandc->vld & ~READ_START_VLD));
1054 nandc_set_reg(nandc, NAND_DEV_CMD1,
1055 (nandc->cmd1 & ~(0xFF << READ_ADDR))
1056 | NAND_CMD_PARAM << READ_ADDR);
1058 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1060 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1061 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1062 nandc_set_read_loc(nandc, 0, 0, 512, 1);
1064 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1065 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1067 nandc->buf_count = 512;
1068 memset(nandc->data_buffer, 0xff, nandc->buf_count);
1070 config_nand_single_cw_page_read(nandc);
1072 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1073 nandc->buf_count, 0);
1075 /* restore CMD1 and VLD regs */
1076 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1077 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1082 /* sets up descriptors for NAND_CMD_ERASE1 */
1083 static int erase_block(struct qcom_nand_host *host, int page_addr)
1085 struct nand_chip *chip = &host->chip;
1086 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1088 nandc_set_reg(nandc, NAND_FLASH_CMD,
1089 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1090 nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1091 nandc_set_reg(nandc, NAND_ADDR1, 0);
1092 nandc_set_reg(nandc, NAND_DEV0_CFG0,
1093 host->cfg0_raw & ~(7 << CW_PER_PAGE));
1094 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1095 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1096 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1097 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1099 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1100 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1101 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1103 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1105 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1106 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1111 /* sets up descriptors for NAND_CMD_READID */
1112 static int read_id(struct qcom_nand_host *host, int column)
1114 struct nand_chip *chip = &host->chip;
1115 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1120 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1121 nandc_set_reg(nandc, NAND_ADDR0, column);
1122 nandc_set_reg(nandc, NAND_ADDR1, 0);
1123 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1124 nandc->props->is_bam ? 0 : DM_EN);
1125 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1127 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1128 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1130 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1135 /* sets up descriptors for NAND_CMD_RESET */
1136 static int reset(struct qcom_nand_host *host)
1138 struct nand_chip *chip = &host->chip;
1139 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1141 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1142 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1144 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1145 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1147 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1152 /* helpers to submit/free our list of dma descriptors */
1153 static int submit_descs(struct qcom_nand_controller *nandc)
1155 struct desc_info *desc;
1156 dma_cookie_t cookie = 0;
1157 struct bam_transaction *bam_txn = nandc->bam_txn;
1160 if (nandc->props->is_bam) {
1161 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1162 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1167 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1168 r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1169 DMA_PREP_INTERRUPT);
1174 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1175 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 0);
1181 list_for_each_entry(desc, &nandc->desc_list, node)
1182 cookie = dmaengine_submit(desc->dma_desc);
1184 if (nandc->props->is_bam) {
1185 dma_async_issue_pending(nandc->tx_chan);
1186 dma_async_issue_pending(nandc->rx_chan);
1188 if (dma_sync_wait(nandc->cmd_chan, cookie) != DMA_COMPLETE)
1191 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1198 static void free_descs(struct qcom_nand_controller *nandc)
1200 struct desc_info *desc, *n;
1202 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1203 list_del(&desc->node);
1205 if (nandc->props->is_bam)
1206 dma_unmap_sg(nandc->dev, desc->bam_sgl,
1207 desc->sgl_cnt, desc->dir);
1209 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1216 /* reset the register read buffer for next NAND operation */
1217 static void clear_read_regs(struct qcom_nand_controller *nandc)
1219 nandc->reg_read_pos = 0;
1220 nandc_read_buffer_sync(nandc, false);
1223 static void pre_command(struct qcom_nand_host *host, int command)
1225 struct nand_chip *chip = &host->chip;
1226 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1228 nandc->buf_count = 0;
1229 nandc->buf_start = 0;
1230 host->use_ecc = false;
1231 host->last_command = command;
1233 clear_read_regs(nandc);
1235 if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1236 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1237 clear_bam_transaction(nandc);
1241 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1242 * privately maintained status byte, this status byte can be read after
1243 * NAND_CMD_STATUS is called
1245 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1247 struct nand_chip *chip = &host->chip;
1248 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1249 struct nand_ecc_ctrl *ecc = &chip->ecc;
1253 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1254 nandc_read_buffer_sync(nandc, true);
1256 for (i = 0; i < num_cw; i++) {
1257 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1259 if (flash_status & FS_MPU_ERR)
1260 host->status &= ~NAND_STATUS_WP;
1262 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1264 FS_DEVICE_STS_ERR)))
1265 host->status |= NAND_STATUS_FAIL;
1269 static void post_command(struct qcom_nand_host *host, int command)
1271 struct nand_chip *chip = &host->chip;
1272 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1275 case NAND_CMD_READID:
1276 nandc_read_buffer_sync(nandc, true);
1277 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1280 case NAND_CMD_PAGEPROG:
1281 case NAND_CMD_ERASE1:
1282 parse_erase_write_errors(host, command);
1290 * Implements chip->cmdfunc. It's only used for a limited set of commands.
1291 * The rest of the commands wouldn't be called by upper layers. For example,
1292 * NAND_CMD_READOOB would never be called because we have our own versions
1293 * of read_oob ops for nand_ecc_ctrl.
1295 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
1296 int column, int page_addr)
1298 struct nand_chip *chip = mtd_to_nand(mtd);
1299 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1300 struct nand_ecc_ctrl *ecc = &chip->ecc;
1301 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1305 pre_command(host, command);
1308 case NAND_CMD_RESET:
1313 case NAND_CMD_READID:
1314 nandc->buf_count = 4;
1315 ret = read_id(host, column);
1319 case NAND_CMD_PARAM:
1320 ret = nandc_param(host);
1324 case NAND_CMD_ERASE1:
1325 ret = erase_block(host, page_addr);
1329 case NAND_CMD_READ0:
1330 /* we read the entire page for now */
1331 WARN_ON(column != 0);
1333 host->use_ecc = true;
1334 set_address(host, 0, page_addr);
1335 update_rw_regs(host, ecc->steps, true);
1338 case NAND_CMD_SEQIN:
1339 WARN_ON(column != 0);
1340 set_address(host, 0, page_addr);
1343 case NAND_CMD_PAGEPROG:
1344 case NAND_CMD_STATUS:
1351 dev_err(nandc->dev, "failure executing command %d\n",
1358 ret = submit_descs(nandc);
1361 "failure submitting descs for command %d\n",
1367 post_command(host, command);
1371 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1372 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1374 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1375 * but it notifies that it is an erased CW by placing special characters at
1376 * certain offsets in the buffer.
1378 * verify if the page is erased or not, and fix up the page for RS ECC by
1379 * replacing the special characters with 0xff.
1381 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1386 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1387 * is erased by looking for 0x54s at offsets 3 and 175 from the
1388 * beginning of each codeword
1391 empty1 = data_buf[3];
1392 empty2 = data_buf[175];
1395 * if the erased codework markers, if they exist override them with
1398 if ((empty1 == 0x54 && empty2 == 0xff) ||
1399 (empty1 == 0xff && empty2 == 0x54)) {
1401 data_buf[175] = 0xff;
1405 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1406 * restore the original values at the special offsets
1408 if (memchr_inv(data_buf, 0xff, data_len)) {
1409 data_buf[3] = empty1;
1410 data_buf[175] = empty2;
1425 * reads back status registers set by the controller to notify page read
1426 * errors. this is equivalent to what 'ecc->correct()' would do.
1428 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1431 struct nand_chip *chip = &host->chip;
1432 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1433 struct mtd_info *mtd = nand_to_mtd(chip);
1434 struct nand_ecc_ctrl *ecc = &chip->ecc;
1435 unsigned int max_bitflips = 0;
1436 struct read_stats *buf;
1439 buf = (struct read_stats *)nandc->reg_read_buf;
1440 nandc_read_buffer_sync(nandc, true);
1442 for (i = 0; i < ecc->steps; i++, buf++) {
1443 u32 flash, buffer, erased_cw;
1444 int data_len, oob_len;
1446 if (i == (ecc->steps - 1)) {
1447 data_len = ecc->size - ((ecc->steps - 1) << 2);
1448 oob_len = ecc->steps << 2;
1450 data_len = host->cw_data;
1454 flash = le32_to_cpu(buf->flash);
1455 buffer = le32_to_cpu(buf->buffer);
1456 erased_cw = le32_to_cpu(buf->erased_cw);
1458 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1461 /* ignore erased codeword errors */
1462 if (host->bch_enabled) {
1463 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1466 erased = erased_chunk_check_and_fixup(data_buf,
1471 data_buf += data_len;
1473 oob_buf += oob_len + ecc->bytes;
1477 if (buffer & BS_UNCORRECTABLE_BIT) {
1478 int ret, ecclen, extraooblen;
1481 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1482 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1483 extraooblen = oob_buf ? oob_len : 0;
1486 * make sure it isn't an erased page reported
1487 * as not-erased by HW because of a few bitflips
1489 ret = nand_check_erased_ecc_chunk(data_buf,
1490 data_len, eccbuf, ecclen, oob_buf,
1491 extraooblen, ecc->strength);
1493 mtd->ecc_stats.failed++;
1495 mtd->ecc_stats.corrected += ret;
1497 max_t(unsigned int, max_bitflips, ret);
1503 stat = buffer & BS_CORRECTABLE_ERR_MSK;
1504 mtd->ecc_stats.corrected += stat;
1505 max_bitflips = max(max_bitflips, stat);
1508 data_buf += data_len;
1510 oob_buf += oob_len + ecc->bytes;
1513 return max_bitflips;
1517 * helper to perform the actual page read operation, used by ecc->read_page(),
1520 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1523 struct nand_chip *chip = &host->chip;
1524 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1525 struct nand_ecc_ctrl *ecc = &chip->ecc;
1528 config_nand_page_read(nandc);
1530 /* queue cmd descs for each codeword */
1531 for (i = 0; i < ecc->steps; i++) {
1532 int data_size, oob_size;
1534 if (i == (ecc->steps - 1)) {
1535 data_size = ecc->size - ((ecc->steps - 1) << 2);
1536 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1539 data_size = host->cw_data;
1540 oob_size = host->ecc_bytes_hw + host->spare_bytes;
1543 if (nandc->props->is_bam) {
1544 if (data_buf && oob_buf) {
1545 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1546 nandc_set_read_loc(nandc, 1, data_size,
1548 } else if (data_buf) {
1549 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1551 nandc_set_read_loc(nandc, 0, data_size,
1556 config_nand_cw_read(nandc);
1559 read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1563 * when ecc is enabled, the controller doesn't read the real
1564 * or dummy bad block markers in each chunk. To maintain a
1565 * consistent layout across RAW and ECC reads, we just
1566 * leave the real/dummy BBM offsets empty (i.e, filled with
1572 for (j = 0; j < host->bbm_size; j++)
1575 read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1576 oob_buf, oob_size, 0);
1580 data_buf += data_size;
1582 oob_buf += oob_size;
1585 ret = submit_descs(nandc);
1587 dev_err(nandc->dev, "failure to read page/oob\n");
1595 * a helper that copies the last step/codeword of a page (containing free oob)
1596 * into our local buffer
1598 static int copy_last_cw(struct qcom_nand_host *host, int page)
1600 struct nand_chip *chip = &host->chip;
1601 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1602 struct nand_ecc_ctrl *ecc = &chip->ecc;
1606 clear_read_regs(nandc);
1608 size = host->use_ecc ? host->cw_data : host->cw_size;
1610 /* prepare a clean read buffer */
1611 memset(nandc->data_buffer, 0xff, size);
1613 set_address(host, host->cw_size * (ecc->steps - 1), page);
1614 update_rw_regs(host, 1, true);
1616 config_nand_single_cw_page_read(nandc);
1618 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1620 ret = submit_descs(nandc);
1622 dev_err(nandc->dev, "failed to copy last codeword\n");
1629 /* implements ecc->read_page() */
1630 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1631 uint8_t *buf, int oob_required, int page)
1633 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1634 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1635 u8 *data_buf, *oob_buf = NULL;
1639 oob_buf = oob_required ? chip->oob_poi : NULL;
1641 clear_bam_transaction(nandc);
1642 ret = read_page_ecc(host, data_buf, oob_buf);
1644 dev_err(nandc->dev, "failure to read page\n");
1648 return parse_read_errors(host, data_buf, oob_buf);
1651 /* implements ecc->read_page_raw() */
1652 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1653 struct nand_chip *chip, uint8_t *buf,
1654 int oob_required, int page)
1656 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1657 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1658 u8 *data_buf, *oob_buf;
1659 struct nand_ecc_ctrl *ecc = &chip->ecc;
1664 oob_buf = chip->oob_poi;
1666 host->use_ecc = false;
1668 clear_bam_transaction(nandc);
1669 update_rw_regs(host, ecc->steps, true);
1670 config_nand_page_read(nandc);
1672 for (i = 0; i < ecc->steps; i++) {
1673 int data_size1, data_size2, oob_size1, oob_size2;
1674 int reg_off = FLASH_BUF_ACC;
1676 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1677 oob_size1 = host->bbm_size;
1679 if (i == (ecc->steps - 1)) {
1680 data_size2 = ecc->size - data_size1 -
1681 ((ecc->steps - 1) << 2);
1682 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1685 data_size2 = host->cw_data - data_size1;
1686 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1689 if (nandc->props->is_bam) {
1691 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1692 read_loc += data_size1;
1694 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1695 read_loc += oob_size1;
1697 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1698 read_loc += data_size2;
1700 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1703 config_nand_cw_read(nandc);
1705 read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1706 reg_off += data_size1;
1707 data_buf += data_size1;
1709 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1710 reg_off += oob_size1;
1711 oob_buf += oob_size1;
1713 read_data_dma(nandc, reg_off, data_buf, data_size2, 0);
1714 reg_off += data_size2;
1715 data_buf += data_size2;
1717 read_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1718 oob_buf += oob_size2;
1721 ret = submit_descs(nandc);
1723 dev_err(nandc->dev, "failure to read raw page\n");
1730 /* implements ecc->read_oob() */
1731 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1734 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1735 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1736 struct nand_ecc_ctrl *ecc = &chip->ecc;
1739 clear_read_regs(nandc);
1740 clear_bam_transaction(nandc);
1742 host->use_ecc = true;
1743 set_address(host, 0, page);
1744 update_rw_regs(host, ecc->steps, true);
1746 ret = read_page_ecc(host, NULL, chip->oob_poi);
1748 dev_err(nandc->dev, "failure to read oob\n");
1753 /* implements ecc->write_page() */
1754 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1755 const uint8_t *buf, int oob_required, int page)
1757 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1758 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1759 struct nand_ecc_ctrl *ecc = &chip->ecc;
1760 u8 *data_buf, *oob_buf;
1763 clear_read_regs(nandc);
1764 clear_bam_transaction(nandc);
1766 data_buf = (u8 *)buf;
1767 oob_buf = chip->oob_poi;
1769 host->use_ecc = true;
1770 update_rw_regs(host, ecc->steps, false);
1771 config_nand_page_write(nandc);
1773 for (i = 0; i < ecc->steps; i++) {
1774 int data_size, oob_size;
1776 if (i == (ecc->steps - 1)) {
1777 data_size = ecc->size - ((ecc->steps - 1) << 2);
1778 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1781 data_size = host->cw_data;
1782 oob_size = ecc->bytes;
1786 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
1787 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
1790 * when ECC is enabled, we don't really need to write anything
1791 * to oob for the first n - 1 codewords since these oob regions
1792 * just contain ECC bytes that's written by the controller
1793 * itself. For the last codeword, we skip the bbm positions and
1794 * write to the free oob area.
1796 if (i == (ecc->steps - 1)) {
1797 oob_buf += host->bbm_size;
1799 write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1800 oob_buf, oob_size, 0);
1803 config_nand_cw_write(nandc);
1805 data_buf += data_size;
1806 oob_buf += oob_size;
1809 ret = submit_descs(nandc);
1811 dev_err(nandc->dev, "failure to write page\n");
1818 /* implements ecc->write_page_raw() */
1819 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1820 struct nand_chip *chip, const uint8_t *buf,
1821 int oob_required, int page)
1823 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1824 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1825 struct nand_ecc_ctrl *ecc = &chip->ecc;
1826 u8 *data_buf, *oob_buf;
1829 clear_read_regs(nandc);
1830 clear_bam_transaction(nandc);
1832 data_buf = (u8 *)buf;
1833 oob_buf = chip->oob_poi;
1835 host->use_ecc = false;
1836 update_rw_regs(host, ecc->steps, false);
1837 config_nand_page_write(nandc);
1839 for (i = 0; i < ecc->steps; i++) {
1840 int data_size1, data_size2, oob_size1, oob_size2;
1841 int reg_off = FLASH_BUF_ACC;
1843 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1844 oob_size1 = host->bbm_size;
1846 if (i == (ecc->steps - 1)) {
1847 data_size2 = ecc->size - data_size1 -
1848 ((ecc->steps - 1) << 2);
1849 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1852 data_size2 = host->cw_data - data_size1;
1853 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1856 write_data_dma(nandc, reg_off, data_buf, data_size1,
1858 reg_off += data_size1;
1859 data_buf += data_size1;
1861 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
1863 reg_off += oob_size1;
1864 oob_buf += oob_size1;
1866 write_data_dma(nandc, reg_off, data_buf, data_size2,
1868 reg_off += data_size2;
1869 data_buf += data_size2;
1871 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1872 oob_buf += oob_size2;
1874 config_nand_cw_write(nandc);
1877 ret = submit_descs(nandc);
1879 dev_err(nandc->dev, "failure to write raw page\n");
1887 * implements ecc->write_oob()
1889 * the NAND controller cannot write only data or only oob within a codeword,
1890 * since ecc is calculated for the combined codeword. we first copy the
1891 * entire contents for the last codeword(data + oob), replace the old oob
1892 * with the new one in chip->oob_poi, and then write the entire codeword.
1893 * this read-copy-write operation results in a slight performance loss.
1895 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1898 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1899 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1900 struct nand_ecc_ctrl *ecc = &chip->ecc;
1901 u8 *oob = chip->oob_poi;
1902 int data_size, oob_size;
1903 int ret, status = 0;
1905 host->use_ecc = true;
1907 clear_bam_transaction(nandc);
1908 ret = copy_last_cw(host, page);
1912 clear_read_regs(nandc);
1913 clear_bam_transaction(nandc);
1915 /* calculate the data and oob size for the last codeword/step */
1916 data_size = ecc->size - ((ecc->steps - 1) << 2);
1917 oob_size = mtd->oobavail;
1919 /* override new oob content to last codeword */
1920 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1923 set_address(host, host->cw_size * (ecc->steps - 1), page);
1924 update_rw_regs(host, 1, false);
1926 config_nand_page_write(nandc);
1927 write_data_dma(nandc, FLASH_BUF_ACC,
1928 nandc->data_buffer, data_size + oob_size, 0);
1929 config_nand_cw_write(nandc);
1931 ret = submit_descs(nandc);
1936 dev_err(nandc->dev, "failure to write oob\n");
1940 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1942 status = chip->waitfunc(mtd, chip);
1944 return status & NAND_STATUS_FAIL ? -EIO : 0;
1947 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1949 struct nand_chip *chip = mtd_to_nand(mtd);
1950 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1951 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1952 struct nand_ecc_ctrl *ecc = &chip->ecc;
1953 int page, ret, bbpos, bad = 0;
1956 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1959 * configure registers for a raw sub page read, the address is set to
1960 * the beginning of the last codeword, we don't care about reading ecc
1961 * portion of oob. we just want the first few bytes from this codeword
1962 * that contains the BBM
1964 host->use_ecc = false;
1966 clear_bam_transaction(nandc);
1967 ret = copy_last_cw(host, page);
1971 flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1973 if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1974 dev_warn(nandc->dev, "error when trying to read BBM\n");
1978 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1980 bad = nandc->data_buffer[bbpos] != 0xff;
1982 if (chip->options & NAND_BUSWIDTH_16)
1983 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1988 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1990 struct nand_chip *chip = mtd_to_nand(mtd);
1991 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1992 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1993 struct nand_ecc_ctrl *ecc = &chip->ecc;
1994 int page, ret, status = 0;
1996 clear_read_regs(nandc);
1997 clear_bam_transaction(nandc);
2000 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2001 * we don't care about the rest of the content in the codeword since
2002 * we aren't going to use this block again
2004 memset(nandc->data_buffer, 0x00, host->cw_size);
2006 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2009 host->use_ecc = false;
2010 set_address(host, host->cw_size * (ecc->steps - 1), page);
2011 update_rw_regs(host, 1, false);
2013 config_nand_page_write(nandc);
2014 write_data_dma(nandc, FLASH_BUF_ACC,
2015 nandc->data_buffer, host->cw_size, 0);
2016 config_nand_cw_write(nandc);
2018 ret = submit_descs(nandc);
2023 dev_err(nandc->dev, "failure to update BBM\n");
2027 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
2029 status = chip->waitfunc(mtd, chip);
2031 return status & NAND_STATUS_FAIL ? -EIO : 0;
2035 * the three functions below implement chip->read_byte(), chip->read_buf()
2036 * and chip->write_buf() respectively. these aren't used for
2037 * reading/writing page data, they are used for smaller data like reading
2040 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
2042 struct nand_chip *chip = mtd_to_nand(mtd);
2043 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2044 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2045 u8 *buf = nandc->data_buffer;
2048 if (host->last_command == NAND_CMD_STATUS) {
2051 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2056 if (nandc->buf_start < nandc->buf_count)
2057 ret = buf[nandc->buf_start++];
2062 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
2064 struct nand_chip *chip = mtd_to_nand(mtd);
2065 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2066 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2068 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2069 nandc->buf_start += real_len;
2072 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
2075 struct nand_chip *chip = mtd_to_nand(mtd);
2076 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2077 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2079 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2081 nandc->buf_start += real_len;
2084 /* we support only one external chip for now */
2085 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
2087 struct nand_chip *chip = mtd_to_nand(mtd);
2088 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2093 dev_warn(nandc->dev, "invalid chip select\n");
2097 * NAND controller page layout info
2099 * Layout with ECC enabled:
2101 * |----------------------| |---------------------------------|
2102 * | xx.......yy| | *********xx.......yy|
2103 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
2104 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
2105 * | xx.......yy| | *********xx.......yy|
2106 * |----------------------| |---------------------------------|
2107 * codeword 1,2..n-1 codeword n
2108 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
2110 * n = Number of codewords in the page
2112 * * = Spare/free bytes
2113 * x = Unused byte(s)
2114 * y = Reserved byte(s)
2116 * 2K page: n = 4, spare = 16 bytes
2117 * 4K page: n = 8, spare = 32 bytes
2118 * 8K page: n = 16, spare = 64 bytes
2120 * the qcom nand controller operates at a sub page/codeword level. each
2121 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2122 * the number of ECC bytes vary based on the ECC strength and the bus width.
2124 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2125 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2126 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2128 * When we access a page with ECC enabled, the reserved bytes(s) are not
2129 * accessible at all. When reading, we fill up these unreadable positions
2130 * with 0xffs. When writing, the controller skips writing the inaccessible
2133 * Layout with ECC disabled:
2135 * |------------------------------| |---------------------------------------|
2136 * | yy xx.......| | bb *********xx.......|
2137 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
2138 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
2139 * | yy xx.......| | bb *********xx.......|
2140 * |------------------------------| |---------------------------------------|
2141 * codeword 1,2..n-1 codeword n
2142 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
2144 * n = Number of codewords in the page
2146 * * = Spare/free bytes
2147 * x = Unused byte(s)
2148 * y = Dummy Bad Bock byte(s)
2149 * b = Real Bad Block byte(s)
2150 * size1/size2 = function of codeword size and 'n'
2152 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2153 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2154 * Block Markers. In the last codeword, this position contains the real BBM
2156 * In order to have a consistent layout between RAW and ECC modes, we assume
2157 * the following OOB layout arrangement:
2159 * |-----------| |--------------------|
2160 * |yyxx.......| |bb*********xx.......|
2161 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
2162 * |yyxx.......| |bb*********xx.......|
2163 * |yyxx.......| |bb*********xx.......|
2164 * |-----------| |--------------------|
2165 * first n - 1 nth OOB region
2168 * n = Number of codewords in the page
2170 * * = FREE OOB bytes
2171 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2172 * x = Unused byte(s)
2173 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2175 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2176 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2177 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2178 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2179 * the sum of the three).
2181 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2182 struct mtd_oob_region *oobregion)
2184 struct nand_chip *chip = mtd_to_nand(mtd);
2185 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2186 struct nand_ecc_ctrl *ecc = &chip->ecc;
2192 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2194 oobregion->offset = 0;
2196 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2197 oobregion->offset = mtd->oobsize - oobregion->length;
2203 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2204 struct mtd_oob_region *oobregion)
2206 struct nand_chip *chip = mtd_to_nand(mtd);
2207 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2208 struct nand_ecc_ctrl *ecc = &chip->ecc;
2213 oobregion->length = ecc->steps * 4;
2214 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2219 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2220 .ecc = qcom_nand_ooblayout_ecc,
2221 .free = qcom_nand_ooblayout_free,
2224 static int qcom_nand_host_setup(struct qcom_nand_host *host)
2226 struct nand_chip *chip = &host->chip;
2227 struct mtd_info *mtd = nand_to_mtd(chip);
2228 struct nand_ecc_ctrl *ecc = &chip->ecc;
2229 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2230 int cwperpage, bad_block_byte;
2235 * the controller requires each step consists of 512 bytes of data.
2236 * bail out if DT has populated a wrong step size.
2238 if (ecc->size != NANDC_STEP_SIZE) {
2239 dev_err(nandc->dev, "invalid ecc size\n");
2243 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2245 if (ecc->strength >= 8) {
2246 /* 8 bit ECC defaults to BCH ECC on all platforms */
2247 host->bch_enabled = true;
2251 host->ecc_bytes_hw = 14;
2252 host->spare_bytes = 0;
2255 host->ecc_bytes_hw = 13;
2256 host->spare_bytes = 2;
2261 * if the controller supports BCH for 4 bit ECC, the controller
2262 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2265 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2267 host->bch_enabled = true;
2271 host->ecc_bytes_hw = 8;
2272 host->spare_bytes = 2;
2275 host->ecc_bytes_hw = 7;
2276 host->spare_bytes = 4;
2281 host->ecc_bytes_hw = 10;
2284 host->spare_bytes = 0;
2287 host->spare_bytes = 1;
2294 * we consider ecc->bytes as the sum of all the non-data content in a
2295 * step. It gives us a clean representation of the oob area (even if
2296 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2297 * ECC and 12 bytes for 4 bit ECC
2299 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2301 ecc->read_page = qcom_nandc_read_page;
2302 ecc->read_page_raw = qcom_nandc_read_page_raw;
2303 ecc->read_oob = qcom_nandc_read_oob;
2304 ecc->write_page = qcom_nandc_write_page;
2305 ecc->write_page_raw = qcom_nandc_write_page_raw;
2306 ecc->write_oob = qcom_nandc_write_oob;
2308 ecc->mode = NAND_ECC_HW;
2310 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2312 cwperpage = mtd->writesize / ecc->size;
2313 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2317 * DATA_UD_BYTES varies based on whether the read/write command protects
2318 * spare data with ECC too. We protect spare data by default, so we set
2319 * it to main + spare data, which are 512 and 4 bytes respectively.
2321 host->cw_data = 516;
2324 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2327 host->cw_size = host->cw_data + ecc->bytes;
2329 if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
2330 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
2334 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2336 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2337 | host->cw_data << UD_SIZE_BYTES
2338 | 0 << DISABLE_STATUS_AFTER_WRITE
2339 | 5 << NUM_ADDR_CYCLES
2340 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2341 | 0 << STATUS_BFR_READ
2342 | 1 << SET_RD_MODE_AFTER_STATUS
2343 | host->spare_bytes << SPARE_SIZE_BYTES;
2345 host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2346 | 0 << CS_ACTIVE_BSY
2347 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2348 | 0 << BAD_BLOCK_IN_SPARE_AREA
2349 | 2 << WR_RD_BSY_GAP
2350 | wide_bus << WIDE_FLASH
2351 | host->bch_enabled << ENABLE_BCH_ECC;
2353 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2354 | host->cw_size << UD_SIZE_BYTES
2355 | 5 << NUM_ADDR_CYCLES
2356 | 0 << SPARE_SIZE_BYTES;
2358 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2359 | 0 << CS_ACTIVE_BSY
2360 | 17 << BAD_BLOCK_BYTE_NUM
2361 | 1 << BAD_BLOCK_IN_SPARE_AREA
2362 | 2 << WR_RD_BSY_GAP
2363 | wide_bus << WIDE_FLASH
2364 | 1 << DEV0_CFG1_ECC_DISABLE;
2366 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2368 | host->cw_data << ECC_NUM_DATA_BYTES
2369 | 1 << ECC_FORCE_CLK_OPEN
2370 | ecc_mode << ECC_MODE
2371 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2373 host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2375 host->clrflashstatus = FS_READY_BSY_N;
2376 host->clrreadstatus = 0xc0;
2377 nandc->regs->erased_cw_detect_cfg_clr =
2378 cpu_to_le32(CLR_ERASED_PAGE_DET);
2379 nandc->regs->erased_cw_detect_cfg_set =
2380 cpu_to_le32(SET_ERASED_PAGE_DET);
2383 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2384 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2385 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2391 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2395 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2397 dev_err(nandc->dev, "failed to set DMA mask\n");
2402 * we use the internal buffer for reading ONFI params, reading small
2403 * data like ID and status, and preforming read-copy-write operations
2404 * when writing to a codeword partially. 532 is the maximum possible
2405 * size of a codeword for our nand controller
2407 nandc->buf_size = 532;
2409 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2411 if (!nandc->data_buffer)
2414 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2419 nandc->reg_read_buf = devm_kzalloc(nandc->dev,
2420 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
2422 if (!nandc->reg_read_buf)
2425 if (nandc->props->is_bam) {
2426 nandc->reg_read_dma =
2427 dma_map_single(nandc->dev, nandc->reg_read_buf,
2429 sizeof(*nandc->reg_read_buf),
2431 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2432 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2436 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2437 if (!nandc->tx_chan) {
2438 dev_err(nandc->dev, "failed to request tx channel\n");
2442 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2443 if (!nandc->rx_chan) {
2444 dev_err(nandc->dev, "failed to request rx channel\n");
2448 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2449 if (!nandc->cmd_chan) {
2450 dev_err(nandc->dev, "failed to request cmd channel\n");
2455 * Initially allocate BAM transaction to read ONFI param page.
2456 * After detecting all the devices, this BAM transaction will
2457 * be freed and the next BAM tranasction will be allocated with
2458 * maximum codeword size
2460 nandc->max_cwperpage = 1;
2461 nandc->bam_txn = alloc_bam_transaction(nandc);
2462 if (!nandc->bam_txn) {
2464 "failed to allocate bam transaction\n");
2468 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2471 "failed to request slave channel\n");
2476 INIT_LIST_HEAD(&nandc->desc_list);
2477 INIT_LIST_HEAD(&nandc->host_list);
2479 nand_hw_control_init(&nandc->controller);
2484 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2486 if (nandc->props->is_bam) {
2487 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2488 dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2490 sizeof(*nandc->reg_read_buf),
2494 dma_release_channel(nandc->tx_chan);
2497 dma_release_channel(nandc->rx_chan);
2499 if (nandc->cmd_chan)
2500 dma_release_channel(nandc->cmd_chan);
2503 dma_release_channel(nandc->chan);
2507 /* one time setup of a few nand controller registers */
2508 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2513 if (!nandc->props->is_qpic)
2514 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2515 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2516 NAND_DEV_CMD_VLD_VAL);
2518 /* enable ADM or BAM DMA */
2519 if (nandc->props->is_bam) {
2520 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2521 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2523 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2526 /* save the original values of these registers */
2527 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2528 nandc->vld = NAND_DEV_CMD_VLD_VAL;
2533 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
2534 struct qcom_nand_host *host,
2535 struct device_node *dn)
2537 struct nand_chip *chip = &host->chip;
2538 struct mtd_info *mtd = nand_to_mtd(chip);
2539 struct device *dev = nandc->dev;
2542 ret = of_property_read_u32(dn, "reg", &host->cs);
2544 dev_err(dev, "can't get chip-select\n");
2548 nand_set_flash_node(chip, dn);
2549 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2553 mtd->owner = THIS_MODULE;
2554 mtd->dev.parent = dev;
2556 chip->cmdfunc = qcom_nandc_command;
2557 chip->select_chip = qcom_nandc_select_chip;
2558 chip->read_byte = qcom_nandc_read_byte;
2559 chip->read_buf = qcom_nandc_read_buf;
2560 chip->write_buf = qcom_nandc_write_buf;
2561 chip->onfi_set_features = nand_onfi_get_set_features_notsupp;
2562 chip->onfi_get_features = nand_onfi_get_set_features_notsupp;
2565 * the bad block marker is readable only when we read the last codeword
2566 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2567 * helpers don't allow us to read BB from a nand chip with ECC
2568 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2569 * and block_markbad helpers until we permanently switch to using
2570 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2572 chip->block_bad = qcom_nandc_block_bad;
2573 chip->block_markbad = qcom_nandc_block_markbad;
2575 chip->controller = &nandc->controller;
2576 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2579 /* set up initial status value */
2580 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2582 ret = nand_scan_ident(mtd, 1, NULL);
2586 ret = qcom_nand_host_setup(host);
2591 static int qcom_nand_mtd_register(struct qcom_nand_controller *nandc,
2592 struct qcom_nand_host *host,
2593 struct device_node *dn)
2595 struct nand_chip *chip = &host->chip;
2596 struct mtd_info *mtd = nand_to_mtd(chip);
2599 ret = nand_scan_tail(mtd);
2603 ret = mtd_device_register(mtd, NULL, 0);
2605 nand_cleanup(mtd_to_nand(mtd));
2610 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2612 struct device *dev = nandc->dev;
2613 struct device_node *dn = dev->of_node, *child;
2614 struct qcom_nand_host *host, *tmp;
2617 for_each_available_child_of_node(dn, child) {
2618 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2624 ret = qcom_nand_host_init(nandc, host, child);
2626 devm_kfree(dev, host);
2630 list_add_tail(&host->node, &nandc->host_list);
2633 if (list_empty(&nandc->host_list))
2636 if (nandc->props->is_bam) {
2637 free_bam_transaction(nandc);
2638 nandc->bam_txn = alloc_bam_transaction(nandc);
2639 if (!nandc->bam_txn) {
2641 "failed to allocate bam transaction\n");
2646 list_for_each_entry_safe(host, tmp, &nandc->host_list, node) {
2647 ret = qcom_nand_mtd_register(nandc, host, child);
2649 list_del(&host->node);
2650 devm_kfree(dev, host);
2654 if (list_empty(&nandc->host_list))
2660 /* parse custom DT properties here */
2661 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2663 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2664 struct device_node *np = nandc->dev->of_node;
2667 if (!nandc->props->is_bam) {
2668 ret = of_property_read_u32(np, "qcom,cmd-crci",
2671 dev_err(nandc->dev, "command CRCI unspecified\n");
2675 ret = of_property_read_u32(np, "qcom,data-crci",
2678 dev_err(nandc->dev, "data CRCI unspecified\n");
2686 static int qcom_nandc_probe(struct platform_device *pdev)
2688 struct qcom_nand_controller *nandc;
2689 const void *dev_data;
2690 struct device *dev = &pdev->dev;
2691 struct resource *res;
2694 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2698 platform_set_drvdata(pdev, nandc);
2701 dev_data = of_device_get_match_data(dev);
2703 dev_err(&pdev->dev, "failed to get device data\n");
2707 nandc->props = dev_data;
2709 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2710 nandc->base = devm_ioremap_resource(dev, res);
2711 if (IS_ERR(nandc->base))
2712 return PTR_ERR(nandc->base);
2714 nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2716 nandc->core_clk = devm_clk_get(dev, "core");
2717 if (IS_ERR(nandc->core_clk))
2718 return PTR_ERR(nandc->core_clk);
2720 nandc->aon_clk = devm_clk_get(dev, "aon");
2721 if (IS_ERR(nandc->aon_clk))
2722 return PTR_ERR(nandc->aon_clk);
2724 ret = qcom_nandc_parse_dt(pdev);
2728 ret = qcom_nandc_alloc(nandc);
2732 ret = clk_prepare_enable(nandc->core_clk);
2736 ret = clk_prepare_enable(nandc->aon_clk);
2740 ret = qcom_nandc_setup(nandc);
2744 ret = qcom_probe_nand_devices(nandc);
2751 clk_disable_unprepare(nandc->aon_clk);
2753 clk_disable_unprepare(nandc->core_clk);
2755 qcom_nandc_unalloc(nandc);
2760 static int qcom_nandc_remove(struct platform_device *pdev)
2762 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2763 struct qcom_nand_host *host;
2765 list_for_each_entry(host, &nandc->host_list, node)
2766 nand_release(&host->chip);
2768 qcom_nandc_unalloc(nandc);
2770 clk_disable_unprepare(nandc->aon_clk);
2771 clk_disable_unprepare(nandc->core_clk);
2776 static const struct qcom_nandc_props ipq806x_nandc_props = {
2777 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
2779 .dev_cmd_reg_start = 0x0,
2782 static const struct qcom_nandc_props ipq4019_nandc_props = {
2783 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2786 .dev_cmd_reg_start = 0x0,
2789 static const struct qcom_nandc_props ipq8074_nandc_props = {
2790 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2793 .dev_cmd_reg_start = 0x7000,
2797 * data will hold a struct pointer containing more differences once we support
2798 * more controller variants
2800 static const struct of_device_id qcom_nandc_of_match[] = {
2802 .compatible = "qcom,ipq806x-nand",
2803 .data = &ipq806x_nandc_props,
2806 .compatible = "qcom,ipq4019-nand",
2807 .data = &ipq4019_nandc_props,
2810 .compatible = "qcom,ipq8074-nand",
2811 .data = &ipq8074_nandc_props,
2815 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2817 static struct platform_driver qcom_nandc_driver = {
2819 .name = "qcom-nandc",
2820 .of_match_table = qcom_nandc_of_match,
2822 .probe = qcom_nandc_probe,
2823 .remove = qcom_nandc_remove,
2825 module_platform_driver(qcom_nandc_driver);
2827 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2828 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2829 MODULE_LICENSE("GPL v2");