2 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
14 #include <linux/clk.h>
15 #include <linux/slab.h>
16 #include <linux/bitops.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmaengine.h>
19 #include <linux/module.h>
20 #include <linux/mtd/rawnand.h>
21 #include <linux/mtd/partitions.h>
23 #include <linux/of_device.h>
24 #include <linux/delay.h>
25 #include <linux/dma/qcom_bam_dma.h>
27 /* NANDc reg offsets */
28 #define NAND_FLASH_CMD 0x00
29 #define NAND_ADDR0 0x04
30 #define NAND_ADDR1 0x08
31 #define NAND_FLASH_CHIP_SELECT 0x0c
32 #define NAND_EXEC_CMD 0x10
33 #define NAND_FLASH_STATUS 0x14
34 #define NAND_BUFFER_STATUS 0x18
35 #define NAND_DEV0_CFG0 0x20
36 #define NAND_DEV0_CFG1 0x24
37 #define NAND_DEV0_ECC_CFG 0x28
38 #define NAND_DEV1_ECC_CFG 0x2c
39 #define NAND_DEV1_CFG0 0x30
40 #define NAND_DEV1_CFG1 0x34
41 #define NAND_READ_ID 0x40
42 #define NAND_READ_STATUS 0x44
43 #define NAND_DEV_CMD0 0xa0
44 #define NAND_DEV_CMD1 0xa4
45 #define NAND_DEV_CMD2 0xa8
46 #define NAND_DEV_CMD_VLD 0xac
47 #define SFLASHC_BURST_CFG 0xe0
48 #define NAND_ERASED_CW_DETECT_CFG 0xe8
49 #define NAND_ERASED_CW_DETECT_STATUS 0xec
50 #define NAND_EBI2_ECC_BUF_CFG 0xf0
51 #define FLASH_BUF_ACC 0x100
53 #define NAND_CTRL 0xf00
54 #define NAND_VERSION 0xf08
55 #define NAND_READ_LOCATION_0 0xf20
56 #define NAND_READ_LOCATION_1 0xf24
57 #define NAND_READ_LOCATION_2 0xf28
58 #define NAND_READ_LOCATION_3 0xf2c
60 /* dummy register offsets, used by write_reg_dma */
61 #define NAND_DEV_CMD1_RESTORE 0xdead
62 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef
64 /* NAND_FLASH_CMD bits */
65 #define PAGE_ACC BIT(4)
66 #define LAST_PAGE BIT(5)
68 /* NAND_FLASH_CHIP_SELECT bits */
69 #define NAND_DEV_SEL 0
72 /* NAND_FLASH_STATUS bits */
73 #define FS_OP_ERR BIT(4)
74 #define FS_READY_BSY_N BIT(5)
75 #define FS_MPU_ERR BIT(8)
76 #define FS_DEVICE_STS_ERR BIT(16)
77 #define FS_DEVICE_WP BIT(23)
79 /* NAND_BUFFER_STATUS bits */
80 #define BS_UNCORRECTABLE_BIT BIT(8)
81 #define BS_CORRECTABLE_ERR_MSK 0x1f
83 /* NAND_DEVn_CFG0 bits */
84 #define DISABLE_STATUS_AFTER_WRITE 4
86 #define UD_SIZE_BYTES 9
87 #define ECC_PARITY_SIZE_BYTES_RS 19
88 #define SPARE_SIZE_BYTES 23
89 #define NUM_ADDR_CYCLES 27
90 #define STATUS_BFR_READ 30
91 #define SET_RD_MODE_AFTER_STATUS 31
93 /* NAND_DEVn_CFG0 bits */
94 #define DEV0_CFG1_ECC_DISABLE 0
96 #define NAND_RECOVERY_CYCLES 2
97 #define CS_ACTIVE_BSY 5
98 #define BAD_BLOCK_BYTE_NUM 6
99 #define BAD_BLOCK_IN_SPARE_AREA 16
100 #define WR_RD_BSY_GAP 17
101 #define ENABLE_BCH_ECC 27
103 /* NAND_DEV0_ECC_CFG bits */
104 #define ECC_CFG_ECC_DISABLE 0
105 #define ECC_SW_RESET 1
107 #define ECC_PARITY_SIZE_BYTES_BCH 8
108 #define ECC_NUM_DATA_BYTES 16
109 #define ECC_FORCE_CLK_OPEN 30
111 /* NAND_DEV_CMD1 bits */
114 /* NAND_DEV_CMD_VLD bits */
115 #define READ_START_VLD BIT(0)
116 #define READ_STOP_VLD BIT(1)
117 #define WRITE_START_VLD BIT(2)
118 #define ERASE_START_VLD BIT(3)
119 #define SEQ_READ_START_VLD BIT(4)
121 /* NAND_EBI2_ECC_BUF_CFG bits */
124 /* NAND_ERASED_CW_DETECT_CFG bits */
125 #define ERASED_CW_ECC_MASK 1
126 #define AUTO_DETECT_RES 0
127 #define MASK_ECC (1 << ERASED_CW_ECC_MASK)
128 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
129 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
130 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
131 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
133 /* NAND_ERASED_CW_DETECT_STATUS bits */
134 #define PAGE_ALL_ERASED BIT(7)
135 #define CODEWORD_ALL_ERASED BIT(6)
136 #define PAGE_ERASED BIT(5)
137 #define CODEWORD_ERASED BIT(4)
138 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
139 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
141 /* NAND_READ_LOCATION_n bits */
142 #define READ_LOCATION_OFFSET 0
143 #define READ_LOCATION_SIZE 16
144 #define READ_LOCATION_LAST 31
147 #define NAND_VERSION_MAJOR_MASK 0xf0000000
148 #define NAND_VERSION_MAJOR_SHIFT 28
149 #define NAND_VERSION_MINOR_MASK 0x0fff0000
150 #define NAND_VERSION_MINOR_SHIFT 16
153 #define OP_PAGE_READ 0x2
154 #define OP_PAGE_READ_WITH_ECC 0x3
155 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4
156 #define OP_PROGRAM_PAGE 0x6
157 #define OP_PAGE_PROGRAM_WITH_ECC 0x7
158 #define OP_PROGRAM_PAGE_SPARE 0x9
159 #define OP_BLOCK_ERASE 0xa
160 #define OP_FETCH_ID 0xb
161 #define OP_RESET_DEVICE 0xd
163 /* Default Value for NAND_DEV_CMD_VLD */
164 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \
165 ERASE_START_VLD | SEQ_READ_START_VLD)
168 #define BAM_MODE_EN BIT(0)
171 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
172 * the driver calls the chunks 'step' or 'codeword' interchangeably
174 #define NANDC_STEP_SIZE 512
177 * the largest page size we support is 8K, this will have 16 steps/codewords
180 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
182 /* we read at most 3 registers per codeword scan */
183 #define MAX_REG_RD (3 * MAX_NUM_STEPS)
185 /* ECC modes supported by the controller */
186 #define ECC_NONE BIT(0)
187 #define ECC_RS_4BIT BIT(1)
188 #define ECC_BCH_4BIT BIT(2)
189 #define ECC_BCH_8BIT BIT(3)
191 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \
192 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \
193 ((offset) << READ_LOCATION_OFFSET) | \
194 ((size) << READ_LOCATION_SIZE) | \
195 ((is_last) << READ_LOCATION_LAST))
198 * Returns the actual register address for all NAND_DEV_ registers
199 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
201 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
203 /* Returns the NAND register physical address */
204 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
206 /* Returns the dma address for reg read buffer */
207 #define reg_buf_dma_addr(chip, vaddr) \
208 ((chip)->reg_read_dma + \
209 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
211 #define QPIC_PER_CW_CMD_ELEMENTS 32
212 #define QPIC_PER_CW_CMD_SGL 32
213 #define QPIC_PER_CW_DATA_SGL 8
215 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000)
218 * Flags used in DMA descriptor preparation helper functions
219 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
221 /* Don't set the EOT in current tx BAM sgl */
222 #define NAND_BAM_NO_EOT BIT(0)
223 /* Set the NWD flag in current BAM sgl */
224 #define NAND_BAM_NWD BIT(1)
225 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
226 #define NAND_BAM_NEXT_SGL BIT(2)
228 * Erased codeword status is being used two times in single transfer so this
229 * flag will determine the current value of erased codeword status register
231 #define NAND_ERASED_CW_SET BIT(4)
234 * This data type corresponds to the BAM transaction which will be used for all
236 * @bam_ce - the array of BAM command elements
237 * @cmd_sgl - sgl for NAND BAM command pipe
238 * @data_sgl - sgl for NAND BAM consumer/producer pipe
239 * @bam_ce_pos - the index in bam_ce which is available for next sgl
240 * @bam_ce_start - the index in bam_ce which marks the start position ce
241 * for current sgl. It will be used for size calculation
243 * @cmd_sgl_pos - current index in command sgl.
244 * @cmd_sgl_start - start index in command sgl.
245 * @tx_sgl_pos - current index in data sgl for tx.
246 * @tx_sgl_start - start index in data sgl for tx.
247 * @rx_sgl_pos - current index in data sgl for rx.
248 * @rx_sgl_start - start index in data sgl for rx.
249 * @wait_second_completion - wait for second DMA desc completion before making
250 * the NAND transfer completion.
251 * @txn_done - completion for NAND transfer.
252 * @last_data_desc - last DMA desc in data channel (tx/rx).
253 * @last_cmd_desc - last DMA desc in command channel.
255 struct bam_transaction {
256 struct bam_cmd_element *bam_ce;
257 struct scatterlist *cmd_sgl;
258 struct scatterlist *data_sgl;
267 bool wait_second_completion;
268 struct completion txn_done;
269 struct dma_async_tx_descriptor *last_data_desc;
270 struct dma_async_tx_descriptor *last_cmd_desc;
274 * This data type corresponds to the nand dma descriptor
275 * @list - list for desc_info
276 * @dir - DMA transfer direction
277 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
279 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
280 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
281 * @dma_desc - low level DMA engine descriptor
284 struct list_head node;
286 enum dma_data_direction dir;
288 struct scatterlist adm_sgl;
290 struct scatterlist *bam_sgl;
294 struct dma_async_tx_descriptor *dma_desc;
298 * holds the current register values that we want to write. acts as a contiguous
299 * chunk of memory which we use to write the controller registers through DMA.
312 __le32 clrflashstatus;
313 __le32 clrreadstatus;
322 __le32 read_location0;
323 __le32 read_location1;
324 __le32 read_location2;
325 __le32 read_location3;
327 __le32 erased_cw_detect_cfg_clr;
328 __le32 erased_cw_detect_cfg_set;
332 * NAND controller data struct
334 * @controller: base controller structure
335 * @host_list: list containing all the chips attached to the
337 * @dev: parent device
339 * @base_phys: physical base address of controller registers
340 * @base_dma: dma base address of controller registers
341 * @core_clk: controller clock
342 * @aon_clk: another controller clock
345 * @cmd_crci: ADM DMA CRCI for command flow control
346 * @data_crci: ADM DMA CRCI for data flow control
347 * @desc_list: DMA descriptor list (list of desc_infos)
349 * @data_buffer: our local DMA buffer for page read/writes,
350 * used when we can't use the buffer provided
351 * by upper layers directly
352 * @buf_size/count/start: markers for chip->read_buf/write_buf functions
353 * @reg_read_buf: local buffer for reading back registers via DMA
354 * @reg_read_dma: contains dma address for register read buffer
355 * @reg_read_pos: marker for data read in reg_read_buf
357 * @regs: a contiguous chunk of memory for DMA register
358 * writes. contains the register values to be
359 * written to controller
360 * @cmd1/vld: some fixed controller register values
361 * @props: properties of current NAND controller,
362 * initialized via DT match data
363 * @max_cwperpage: maximum QPIC codewords required. calculated
364 * from all connected NAND devices pagesize
366 struct qcom_nand_controller {
367 struct nand_controller controller;
368 struct list_head host_list;
373 phys_addr_t base_phys;
376 struct clk *core_clk;
380 /* will be used only by QPIC for BAM DMA */
382 struct dma_chan *tx_chan;
383 struct dma_chan *rx_chan;
384 struct dma_chan *cmd_chan;
387 /* will be used only by EBI2 for ADM DMA */
389 struct dma_chan *chan;
390 unsigned int cmd_crci;
391 unsigned int data_crci;
395 struct list_head desc_list;
396 struct bam_transaction *bam_txn;
402 unsigned int max_cwperpage;
404 __le32 *reg_read_buf;
405 dma_addr_t reg_read_dma;
408 struct nandc_regs *regs;
411 const struct qcom_nandc_props *props;
415 * NAND chip structure
417 * @chip: base NAND chip structure
418 * @node: list node to add itself to host_list in
419 * qcom_nand_controller
421 * @cs: chip select value for this chip
422 * @cw_size: the number of bytes in a single step/codeword
423 * of a page, consisting of all data, ecc, spare
425 * @cw_data: the number of bytes within a codeword protected
427 * @use_ecc: request the controller to use ECC for the
428 * upcoming read/write
429 * @bch_enabled: flag to tell whether BCH ECC mode is used
430 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
432 * @status: value to be returned if NAND_CMD_STATUS command
434 * @last_command: keeps track of last command on this chip. used
435 * for reading correct status
437 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
438 * ecc/non-ecc mode for the current nand flash
441 struct qcom_nand_host {
442 struct nand_chip chip;
443 struct list_head node;
457 u32 cfg0_raw, cfg1_raw;
465 * This data type corresponds to the NAND controller properties which varies
466 * among different NAND controllers.
467 * @ecc_modes - ecc mode for NAND
468 * @is_bam - whether NAND controller is using BAM
469 * @is_qpic - whether NAND CTRL is part of qpic IP
470 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
472 struct qcom_nandc_props {
476 u32 dev_cmd_reg_start;
479 /* Frees the BAM transaction memory */
480 static void free_bam_transaction(struct qcom_nand_controller *nandc)
482 struct bam_transaction *bam_txn = nandc->bam_txn;
484 devm_kfree(nandc->dev, bam_txn);
487 /* Allocates and Initializes the BAM transaction */
488 static struct bam_transaction *
489 alloc_bam_transaction(struct qcom_nand_controller *nandc)
491 struct bam_transaction *bam_txn;
493 unsigned int num_cw = nandc->max_cwperpage;
497 sizeof(*bam_txn) + num_cw *
498 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
499 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
500 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
502 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
506 bam_txn = bam_txn_buf;
507 bam_txn_buf += sizeof(*bam_txn);
509 bam_txn->bam_ce = bam_txn_buf;
511 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
513 bam_txn->cmd_sgl = bam_txn_buf;
515 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
517 bam_txn->data_sgl = bam_txn_buf;
519 init_completion(&bam_txn->txn_done);
524 /* Clears the BAM transaction indexes */
525 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
527 struct bam_transaction *bam_txn = nandc->bam_txn;
529 if (!nandc->props->is_bam)
532 bam_txn->bam_ce_pos = 0;
533 bam_txn->bam_ce_start = 0;
534 bam_txn->cmd_sgl_pos = 0;
535 bam_txn->cmd_sgl_start = 0;
536 bam_txn->tx_sgl_pos = 0;
537 bam_txn->tx_sgl_start = 0;
538 bam_txn->rx_sgl_pos = 0;
539 bam_txn->rx_sgl_start = 0;
540 bam_txn->last_data_desc = NULL;
541 bam_txn->wait_second_completion = false;
543 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
544 QPIC_PER_CW_CMD_SGL);
545 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
546 QPIC_PER_CW_DATA_SGL);
548 reinit_completion(&bam_txn->txn_done);
551 /* Callback for DMA descriptor completion */
552 static void qpic_bam_dma_done(void *data)
554 struct bam_transaction *bam_txn = data;
557 * In case of data transfer with NAND, 2 callbacks will be generated.
558 * One for command channel and another one for data channel.
559 * If current transaction has data descriptors
560 * (i.e. wait_second_completion is true), then set this to false
561 * and wait for second DMA descriptor completion.
563 if (bam_txn->wait_second_completion)
564 bam_txn->wait_second_completion = false;
566 complete(&bam_txn->txn_done);
569 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
571 return container_of(chip, struct qcom_nand_host, chip);
574 static inline struct qcom_nand_controller *
575 get_qcom_nand_controller(struct nand_chip *chip)
577 return container_of(chip->controller, struct qcom_nand_controller,
581 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
583 return ioread32(nandc->base + offset);
586 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
589 iowrite32(val, nandc->base + offset);
592 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
595 if (!nandc->props->is_bam)
599 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
601 sizeof(*nandc->reg_read_buf),
604 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
606 sizeof(*nandc->reg_read_buf),
610 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
619 case NAND_FLASH_CHIP_SELECT:
620 return ®s->chip_sel;
623 case NAND_FLASH_STATUS:
624 return ®s->clrflashstatus;
629 case NAND_DEV0_ECC_CFG:
630 return ®s->ecc_bch_cfg;
631 case NAND_READ_STATUS:
632 return ®s->clrreadstatus;
635 case NAND_DEV_CMD1_RESTORE:
636 return ®s->orig_cmd1;
637 case NAND_DEV_CMD_VLD:
639 case NAND_DEV_CMD_VLD_RESTORE:
640 return ®s->orig_vld;
641 case NAND_EBI2_ECC_BUF_CFG:
642 return ®s->ecc_buf_cfg;
643 case NAND_READ_LOCATION_0:
644 return ®s->read_location0;
645 case NAND_READ_LOCATION_1:
646 return ®s->read_location1;
647 case NAND_READ_LOCATION_2:
648 return ®s->read_location2;
649 case NAND_READ_LOCATION_3:
650 return ®s->read_location3;
656 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
659 struct nandc_regs *regs = nandc->regs;
662 reg = offset_to_nandc_reg(regs, offset);
665 *reg = cpu_to_le32(val);
668 /* helper to configure address register values */
669 static void set_address(struct qcom_nand_host *host, u16 column, int page)
671 struct nand_chip *chip = &host->chip;
672 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
674 if (chip->options & NAND_BUSWIDTH_16)
677 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
678 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
682 * update_rw_regs: set up read/write register values, these will be
683 * written to the NAND controller registers via DMA
685 * @num_cw: number of steps for the read/write operation
686 * @read: read or write operation
688 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
690 struct nand_chip *chip = &host->chip;
691 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
692 u32 cmd, cfg0, cfg1, ecc_bch_cfg;
696 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
698 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
700 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
704 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
705 (num_cw - 1) << CW_PER_PAGE;
708 ecc_bch_cfg = host->ecc_bch_cfg;
710 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
711 (num_cw - 1) << CW_PER_PAGE;
713 cfg1 = host->cfg1_raw;
714 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
717 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
718 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
719 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
720 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
721 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
722 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
723 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
724 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
727 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
728 host->cw_data : host->cw_size, 1);
732 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
733 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
734 * which will be submitted to DMA engine.
736 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
737 struct dma_chan *chan,
740 struct desc_info *desc;
741 struct scatterlist *sgl;
742 unsigned int sgl_cnt;
744 struct bam_transaction *bam_txn = nandc->bam_txn;
745 enum dma_transfer_direction dir_eng;
746 struct dma_async_tx_descriptor *dma_desc;
748 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
752 if (chan == nandc->cmd_chan) {
753 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
754 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
755 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
756 dir_eng = DMA_MEM_TO_DEV;
757 desc->dir = DMA_TO_DEVICE;
758 } else if (chan == nandc->tx_chan) {
759 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
760 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
761 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
762 dir_eng = DMA_MEM_TO_DEV;
763 desc->dir = DMA_TO_DEVICE;
765 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
766 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
767 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
768 dir_eng = DMA_DEV_TO_MEM;
769 desc->dir = DMA_FROM_DEVICE;
772 sg_mark_end(sgl + sgl_cnt - 1);
773 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
775 dev_err(nandc->dev, "failure in mapping desc\n");
780 desc->sgl_cnt = sgl_cnt;
783 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
787 dev_err(nandc->dev, "failure in prep desc\n");
788 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
793 desc->dma_desc = dma_desc;
795 /* update last data/command descriptor */
796 if (chan == nandc->cmd_chan)
797 bam_txn->last_cmd_desc = dma_desc;
799 bam_txn->last_data_desc = dma_desc;
801 list_add_tail(&desc->node, &nandc->desc_list);
807 * Prepares the command descriptor for BAM DMA which will be used for NAND
808 * register reads and writes. The command descriptor requires the command
809 * to be formed in command element type so this function uses the command
810 * element from bam transaction ce array and fills the same with required
811 * data. A single SGL can contain multiple command elements so
812 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
813 * after the current command element.
815 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
816 int reg_off, const void *vaddr,
817 int size, unsigned int flags)
821 struct bam_cmd_element *bam_ce_buffer;
822 struct bam_transaction *bam_txn = nandc->bam_txn;
824 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
826 /* fill the command desc */
827 for (i = 0; i < size; i++) {
829 bam_prep_ce(&bam_ce_buffer[i],
830 nandc_reg_phys(nandc, reg_off + 4 * i),
832 reg_buf_dma_addr(nandc,
833 (__le32 *)vaddr + i));
835 bam_prep_ce_le32(&bam_ce_buffer[i],
836 nandc_reg_phys(nandc, reg_off + 4 * i),
838 *((__le32 *)vaddr + i));
841 bam_txn->bam_ce_pos += size;
843 /* use the separate sgl after this command */
844 if (flags & NAND_BAM_NEXT_SGL) {
845 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
846 bam_ce_size = (bam_txn->bam_ce_pos -
847 bam_txn->bam_ce_start) *
848 sizeof(struct bam_cmd_element);
849 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
850 bam_ce_buffer, bam_ce_size);
851 bam_txn->cmd_sgl_pos++;
852 bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
854 if (flags & NAND_BAM_NWD) {
855 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
867 * Prepares the data descriptor for BAM DMA which will be used for NAND
868 * data reads and writes.
870 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
872 int size, unsigned int flags)
875 struct bam_transaction *bam_txn = nandc->bam_txn;
878 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
880 bam_txn->rx_sgl_pos++;
882 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
884 bam_txn->tx_sgl_pos++;
887 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
888 * is not set, form the DMA descriptor
890 if (!(flags & NAND_BAM_NO_EOT)) {
891 ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
901 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
902 int reg_off, const void *vaddr, int size,
905 struct desc_info *desc;
906 struct dma_async_tx_descriptor *dma_desc;
907 struct scatterlist *sgl;
908 struct dma_slave_config slave_conf;
909 enum dma_transfer_direction dir_eng;
912 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
916 sgl = &desc->adm_sgl;
918 sg_init_one(sgl, vaddr, size);
921 dir_eng = DMA_DEV_TO_MEM;
922 desc->dir = DMA_FROM_DEVICE;
924 dir_eng = DMA_MEM_TO_DEV;
925 desc->dir = DMA_TO_DEVICE;
928 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
934 memset(&slave_conf, 0x00, sizeof(slave_conf));
936 slave_conf.device_fc = flow_control;
938 slave_conf.src_maxburst = 16;
939 slave_conf.src_addr = nandc->base_dma + reg_off;
940 slave_conf.slave_id = nandc->data_crci;
942 slave_conf.dst_maxburst = 16;
943 slave_conf.dst_addr = nandc->base_dma + reg_off;
944 slave_conf.slave_id = nandc->cmd_crci;
947 ret = dmaengine_slave_config(nandc->chan, &slave_conf);
949 dev_err(nandc->dev, "failed to configure dma channel\n");
953 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
955 dev_err(nandc->dev, "failed to prepare desc\n");
960 desc->dma_desc = dma_desc;
962 list_add_tail(&desc->node, &nandc->desc_list);
972 * read_reg_dma: prepares a descriptor to read a given number of
973 * contiguous registers to the reg_read_buf pointer
975 * @first: offset of the first register in the contiguous block
976 * @num_regs: number of registers to read
977 * @flags: flags to control DMA descriptor preparation
979 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
980 int num_regs, unsigned int flags)
982 bool flow_control = false;
985 vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
986 nandc->reg_read_pos += num_regs;
988 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
989 first = dev_cmd_reg_addr(nandc, first);
991 if (nandc->props->is_bam)
992 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
995 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
998 return prep_adm_dma_desc(nandc, true, first, vaddr,
999 num_regs * sizeof(u32), flow_control);
1003 * write_reg_dma: prepares a descriptor to write a given number of
1004 * contiguous registers
1006 * @first: offset of the first register in the contiguous block
1007 * @num_regs: number of registers to write
1008 * @flags: flags to control DMA descriptor preparation
1010 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1011 int num_regs, unsigned int flags)
1013 bool flow_control = false;
1014 struct nandc_regs *regs = nandc->regs;
1017 vaddr = offset_to_nandc_reg(regs, first);
1019 if (first == NAND_ERASED_CW_DETECT_CFG) {
1020 if (flags & NAND_ERASED_CW_SET)
1021 vaddr = ®s->erased_cw_detect_cfg_set;
1023 vaddr = ®s->erased_cw_detect_cfg_clr;
1026 if (first == NAND_EXEC_CMD)
1027 flags |= NAND_BAM_NWD;
1029 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1030 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1032 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1033 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1035 if (nandc->props->is_bam)
1036 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1039 if (first == NAND_FLASH_CMD)
1040 flow_control = true;
1042 return prep_adm_dma_desc(nandc, false, first, vaddr,
1043 num_regs * sizeof(u32), flow_control);
1047 * read_data_dma: prepares a DMA descriptor to transfer data from the
1048 * controller's internal buffer to the buffer 'vaddr'
1050 * @reg_off: offset within the controller's data buffer
1051 * @vaddr: virtual address of the buffer we want to write to
1052 * @size: DMA transaction size in bytes
1053 * @flags: flags to control DMA descriptor preparation
1055 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1056 const u8 *vaddr, int size, unsigned int flags)
1058 if (nandc->props->is_bam)
1059 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1061 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1065 * write_data_dma: prepares a DMA descriptor to transfer data from
1066 * 'vaddr' to the controller's internal buffer
1068 * @reg_off: offset within the controller's data buffer
1069 * @vaddr: virtual address of the buffer we want to read from
1070 * @size: DMA transaction size in bytes
1071 * @flags: flags to control DMA descriptor preparation
1073 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1074 const u8 *vaddr, int size, unsigned int flags)
1076 if (nandc->props->is_bam)
1077 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1079 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1083 * Helper to prepare DMA descriptors for configuring registers
1084 * before reading a NAND page.
1086 static void config_nand_page_read(struct qcom_nand_controller *nandc)
1088 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1089 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1090 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1091 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1092 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1093 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1097 * Helper to prepare DMA descriptors for configuring registers
1098 * before reading each codeword in NAND page.
1101 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc)
1103 if (nandc->props->is_bam)
1104 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
1107 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1108 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1111 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1112 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1115 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1120 * Helper to prepare dma descriptors to configure registers needed for reading a
1121 * single codeword in page
1124 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc,
1127 config_nand_page_read(nandc);
1128 config_nand_cw_read(nandc, use_ecc);
1132 * Helper to prepare DMA descriptors used to configure registers needed for
1133 * before writing a NAND page.
1135 static void config_nand_page_write(struct qcom_nand_controller *nandc)
1137 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1138 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1139 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1144 * Helper to prepare DMA descriptors for configuring registers
1145 * before writing each codeword in NAND page.
1147 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1149 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1150 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1152 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1154 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1155 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1159 * the following functions are used within chip->cmdfunc() to perform different
1160 * NAND_CMD_* commands
1163 /* sets up descriptors for NAND_CMD_PARAM */
1164 static int nandc_param(struct qcom_nand_host *host)
1166 struct nand_chip *chip = &host->chip;
1167 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1170 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1171 * in use. we configure the controller to perform a raw read of 512
1172 * bytes to read onfi params
1174 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1175 nandc_set_reg(nandc, NAND_ADDR0, 0);
1176 nandc_set_reg(nandc, NAND_ADDR1, 0);
1177 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1178 | 512 << UD_SIZE_BYTES
1179 | 5 << NUM_ADDR_CYCLES
1180 | 0 << SPARE_SIZE_BYTES);
1181 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1182 | 0 << CS_ACTIVE_BSY
1183 | 17 << BAD_BLOCK_BYTE_NUM
1184 | 1 << BAD_BLOCK_IN_SPARE_AREA
1185 | 2 << WR_RD_BSY_GAP
1187 | 1 << DEV0_CFG1_ECC_DISABLE);
1188 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1190 /* configure CMD1 and VLD for ONFI param probing */
1191 nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1192 (nandc->vld & ~READ_START_VLD));
1193 nandc_set_reg(nandc, NAND_DEV_CMD1,
1194 (nandc->cmd1 & ~(0xFF << READ_ADDR))
1195 | NAND_CMD_PARAM << READ_ADDR);
1197 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1199 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1200 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1201 nandc_set_read_loc(nandc, 0, 0, 512, 1);
1203 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1204 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1206 nandc->buf_count = 512;
1207 memset(nandc->data_buffer, 0xff, nandc->buf_count);
1209 config_nand_single_cw_page_read(nandc, false);
1211 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1212 nandc->buf_count, 0);
1214 /* restore CMD1 and VLD regs */
1215 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1216 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1221 /* sets up descriptors for NAND_CMD_ERASE1 */
1222 static int erase_block(struct qcom_nand_host *host, int page_addr)
1224 struct nand_chip *chip = &host->chip;
1225 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1227 nandc_set_reg(nandc, NAND_FLASH_CMD,
1228 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1229 nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1230 nandc_set_reg(nandc, NAND_ADDR1, 0);
1231 nandc_set_reg(nandc, NAND_DEV0_CFG0,
1232 host->cfg0_raw & ~(7 << CW_PER_PAGE));
1233 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1234 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1235 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1236 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1238 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1239 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1240 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1242 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1244 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1245 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1250 /* sets up descriptors for NAND_CMD_READID */
1251 static int read_id(struct qcom_nand_host *host, int column)
1253 struct nand_chip *chip = &host->chip;
1254 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1259 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1260 nandc_set_reg(nandc, NAND_ADDR0, column);
1261 nandc_set_reg(nandc, NAND_ADDR1, 0);
1262 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1263 nandc->props->is_bam ? 0 : DM_EN);
1264 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1266 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1267 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1269 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1274 /* sets up descriptors for NAND_CMD_RESET */
1275 static int reset(struct qcom_nand_host *host)
1277 struct nand_chip *chip = &host->chip;
1278 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1280 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1281 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1283 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1284 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1286 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1291 /* helpers to submit/free our list of dma descriptors */
1292 static int submit_descs(struct qcom_nand_controller *nandc)
1294 struct desc_info *desc;
1295 dma_cookie_t cookie = 0;
1296 struct bam_transaction *bam_txn = nandc->bam_txn;
1299 if (nandc->props->is_bam) {
1300 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1301 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1306 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1307 r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1308 DMA_PREP_INTERRUPT);
1313 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1314 r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1321 list_for_each_entry(desc, &nandc->desc_list, node)
1322 cookie = dmaengine_submit(desc->dma_desc);
1324 if (nandc->props->is_bam) {
1325 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1326 bam_txn->last_cmd_desc->callback_param = bam_txn;
1327 if (bam_txn->last_data_desc) {
1328 bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1329 bam_txn->last_data_desc->callback_param = bam_txn;
1330 bam_txn->wait_second_completion = true;
1333 dma_async_issue_pending(nandc->tx_chan);
1334 dma_async_issue_pending(nandc->rx_chan);
1335 dma_async_issue_pending(nandc->cmd_chan);
1337 if (!wait_for_completion_timeout(&bam_txn->txn_done,
1338 QPIC_NAND_COMPLETION_TIMEOUT))
1341 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1348 static void free_descs(struct qcom_nand_controller *nandc)
1350 struct desc_info *desc, *n;
1352 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1353 list_del(&desc->node);
1355 if (nandc->props->is_bam)
1356 dma_unmap_sg(nandc->dev, desc->bam_sgl,
1357 desc->sgl_cnt, desc->dir);
1359 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1366 /* reset the register read buffer for next NAND operation */
1367 static void clear_read_regs(struct qcom_nand_controller *nandc)
1369 nandc->reg_read_pos = 0;
1370 nandc_read_buffer_sync(nandc, false);
1373 static void pre_command(struct qcom_nand_host *host, int command)
1375 struct nand_chip *chip = &host->chip;
1376 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1378 nandc->buf_count = 0;
1379 nandc->buf_start = 0;
1380 host->use_ecc = false;
1381 host->last_command = command;
1383 clear_read_regs(nandc);
1385 if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1386 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1387 clear_bam_transaction(nandc);
1391 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1392 * privately maintained status byte, this status byte can be read after
1393 * NAND_CMD_STATUS is called
1395 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1397 struct nand_chip *chip = &host->chip;
1398 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1399 struct nand_ecc_ctrl *ecc = &chip->ecc;
1403 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1404 nandc_read_buffer_sync(nandc, true);
1406 for (i = 0; i < num_cw; i++) {
1407 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1409 if (flash_status & FS_MPU_ERR)
1410 host->status &= ~NAND_STATUS_WP;
1412 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1414 FS_DEVICE_STS_ERR)))
1415 host->status |= NAND_STATUS_FAIL;
1419 static void post_command(struct qcom_nand_host *host, int command)
1421 struct nand_chip *chip = &host->chip;
1422 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1425 case NAND_CMD_READID:
1426 nandc_read_buffer_sync(nandc, true);
1427 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1430 case NAND_CMD_PAGEPROG:
1431 case NAND_CMD_ERASE1:
1432 parse_erase_write_errors(host, command);
1440 * Implements chip->cmdfunc. It's only used for a limited set of commands.
1441 * The rest of the commands wouldn't be called by upper layers. For example,
1442 * NAND_CMD_READOOB would never be called because we have our own versions
1443 * of read_oob ops for nand_ecc_ctrl.
1445 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
1446 int column, int page_addr)
1448 struct nand_chip *chip = mtd_to_nand(mtd);
1449 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1450 struct nand_ecc_ctrl *ecc = &chip->ecc;
1451 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1455 pre_command(host, command);
1458 case NAND_CMD_RESET:
1463 case NAND_CMD_READID:
1464 nandc->buf_count = 4;
1465 ret = read_id(host, column);
1469 case NAND_CMD_PARAM:
1470 ret = nandc_param(host);
1474 case NAND_CMD_ERASE1:
1475 ret = erase_block(host, page_addr);
1479 case NAND_CMD_READ0:
1480 /* we read the entire page for now */
1481 WARN_ON(column != 0);
1483 host->use_ecc = true;
1484 set_address(host, 0, page_addr);
1485 update_rw_regs(host, ecc->steps, true);
1488 case NAND_CMD_SEQIN:
1489 WARN_ON(column != 0);
1490 set_address(host, 0, page_addr);
1493 case NAND_CMD_PAGEPROG:
1494 case NAND_CMD_STATUS:
1501 dev_err(nandc->dev, "failure executing command %d\n",
1508 ret = submit_descs(nandc);
1511 "failure submitting descs for command %d\n",
1517 post_command(host, command);
1521 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1522 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1524 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1525 * but it notifies that it is an erased CW by placing special characters at
1526 * certain offsets in the buffer.
1528 * verify if the page is erased or not, and fix up the page for RS ECC by
1529 * replacing the special characters with 0xff.
1531 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1536 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1537 * is erased by looking for 0x54s at offsets 3 and 175 from the
1538 * beginning of each codeword
1541 empty1 = data_buf[3];
1542 empty2 = data_buf[175];
1545 * if the erased codework markers, if they exist override them with
1548 if ((empty1 == 0x54 && empty2 == 0xff) ||
1549 (empty1 == 0xff && empty2 == 0x54)) {
1551 data_buf[175] = 0xff;
1555 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1556 * restore the original values at the special offsets
1558 if (memchr_inv(data_buf, 0xff, data_len)) {
1559 data_buf[3] = empty1;
1560 data_buf[175] = empty2;
1574 /* reads back FLASH_STATUS register set by the controller */
1575 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1577 struct nand_chip *chip = &host->chip;
1578 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1581 nandc_read_buffer_sync(nandc, true);
1583 for (i = 0; i < cw_cnt; i++) {
1584 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1586 if (flash & (FS_OP_ERR | FS_MPU_ERR))
1593 /* performs raw read for one codeword */
1595 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1596 u8 *data_buf, u8 *oob_buf, int page, int cw)
1598 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1599 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1600 struct nand_ecc_ctrl *ecc = &chip->ecc;
1601 int data_size1, data_size2, oob_size1, oob_size2;
1602 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1604 nand_read_page_op(chip, page, 0, NULL, 0);
1605 host->use_ecc = false;
1607 clear_bam_transaction(nandc);
1608 set_address(host, host->cw_size * cw, page);
1609 update_rw_regs(host, 1, true);
1610 config_nand_page_read(nandc);
1612 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1613 oob_size1 = host->bbm_size;
1615 if (cw == (ecc->steps - 1)) {
1616 data_size2 = ecc->size - data_size1 -
1617 ((ecc->steps - 1) * 4);
1618 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1621 data_size2 = host->cw_data - data_size1;
1622 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1625 if (nandc->props->is_bam) {
1626 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1627 read_loc += data_size1;
1629 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1630 read_loc += oob_size1;
1632 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1633 read_loc += data_size2;
1635 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1638 config_nand_cw_read(nandc, false);
1640 read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1641 reg_off += data_size1;
1643 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1644 reg_off += oob_size1;
1646 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1647 reg_off += data_size2;
1649 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1651 ret = submit_descs(nandc);
1654 dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1658 return check_flash_errors(host, 1);
1662 * Bitflips can happen in erased codewords also so this function counts the
1663 * number of 0 in each CW for which ECC engine returns the uncorrectable
1664 * error. The page will be assumed as erased if this count is less than or
1665 * equal to the ecc->strength for each CW.
1667 * 1. Both DATA and OOB need to be checked for number of 0. The
1668 * top-level API can be called with only data buf or OOB buf so use
1669 * chip->data_buf if data buf is null and chip->oob_poi if oob buf
1670 * is null for copying the raw bytes.
1671 * 2. Perform raw read for all the CW which has uncorrectable errors.
1672 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1673 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1674 * the number of bitflips in this area.
1677 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1678 u8 *oob_buf, unsigned long uncorrectable_cws,
1679 int page, unsigned int max_bitflips)
1681 struct nand_chip *chip = &host->chip;
1682 struct mtd_info *mtd = nand_to_mtd(chip);
1683 struct nand_ecc_ctrl *ecc = &chip->ecc;
1684 u8 *cw_data_buf, *cw_oob_buf;
1685 int cw, data_size, oob_size, ret = 0;
1688 data_buf = chip->data_buf;
1693 oob_buf = chip->oob_poi;
1697 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1698 if (cw == (ecc->steps - 1)) {
1699 data_size = ecc->size - ((ecc->steps - 1) * 4);
1700 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1702 data_size = host->cw_data;
1703 oob_size = host->ecc_bytes_hw;
1706 /* determine starting buffer address for current CW */
1707 cw_data_buf = data_buf + (cw * host->cw_data);
1708 cw_oob_buf = oob_buf + (cw * ecc->bytes);
1710 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1711 cw_oob_buf, page, cw);
1716 * make sure it isn't an erased page reported
1717 * as not-erased by HW because of a few bitflips
1719 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1720 cw_oob_buf + host->bbm_size,
1724 mtd->ecc_stats.failed++;
1726 mtd->ecc_stats.corrected += ret;
1727 max_bitflips = max_t(unsigned int, max_bitflips, ret);
1731 return max_bitflips;
1735 * reads back status registers set by the controller to notify page read
1736 * errors. this is equivalent to what 'ecc->correct()' would do.
1738 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1739 u8 *oob_buf, int page)
1741 struct nand_chip *chip = &host->chip;
1742 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1743 struct mtd_info *mtd = nand_to_mtd(chip);
1744 struct nand_ecc_ctrl *ecc = &chip->ecc;
1745 unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1746 struct read_stats *buf;
1747 bool flash_op_err = false, erased;
1749 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1751 buf = (struct read_stats *)nandc->reg_read_buf;
1752 nandc_read_buffer_sync(nandc, true);
1754 for (i = 0; i < ecc->steps; i++, buf++) {
1755 u32 flash, buffer, erased_cw;
1756 int data_len, oob_len;
1758 if (i == (ecc->steps - 1)) {
1759 data_len = ecc->size - ((ecc->steps - 1) << 2);
1760 oob_len = ecc->steps << 2;
1762 data_len = host->cw_data;
1766 flash = le32_to_cpu(buf->flash);
1767 buffer = le32_to_cpu(buf->buffer);
1768 erased_cw = le32_to_cpu(buf->erased_cw);
1771 * Check ECC failure for each codeword. ECC failure can
1772 * happen in either of the following conditions
1773 * 1. If number of bitflips are greater than ECC engine
1775 * 2. If this codeword contains all 0xff for which erased
1776 * codeword detection check will be done.
1778 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1780 * For BCH ECC, ignore erased codeword errors, if
1781 * ERASED_CW bits are set.
1783 if (host->bch_enabled) {
1784 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1787 * For RS ECC, HW reports the erased CW by placing
1788 * special characters at certain offsets in the buffer.
1789 * These special characters will be valid only if
1790 * complete page is read i.e. data_buf is not NULL.
1792 } else if (data_buf) {
1793 erased = erased_chunk_check_and_fixup(data_buf,
1800 uncorrectable_cws |= BIT(i);
1802 * Check if MPU or any other operational error (timeout,
1803 * device failure, etc.) happened for this codeword and
1804 * make flash_op_err true. If flash_op_err is set, then
1805 * EIO will be returned for page read.
1807 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1808 flash_op_err = true;
1810 * No ECC or operational errors happened. Check the number of
1811 * bits corrected and update the ecc_stats.corrected.
1816 stat = buffer & BS_CORRECTABLE_ERR_MSK;
1817 mtd->ecc_stats.corrected += stat;
1818 max_bitflips = max(max_bitflips, stat);
1822 data_buf += data_len;
1824 oob_buf += oob_len + ecc->bytes;
1830 if (!uncorrectable_cws)
1831 return max_bitflips;
1833 return check_for_erased_page(host, data_buf_start, oob_buf_start,
1834 uncorrectable_cws, page,
1839 * helper to perform the actual page read operation, used by ecc->read_page(),
1842 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1843 u8 *oob_buf, int page)
1845 struct nand_chip *chip = &host->chip;
1846 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1847 struct nand_ecc_ctrl *ecc = &chip->ecc;
1848 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1851 config_nand_page_read(nandc);
1853 /* queue cmd descs for each codeword */
1854 for (i = 0; i < ecc->steps; i++) {
1855 int data_size, oob_size;
1857 if (i == (ecc->steps - 1)) {
1858 data_size = ecc->size - ((ecc->steps - 1) << 2);
1859 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1862 data_size = host->cw_data;
1863 oob_size = host->ecc_bytes_hw + host->spare_bytes;
1866 if (nandc->props->is_bam) {
1867 if (data_buf && oob_buf) {
1868 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1869 nandc_set_read_loc(nandc, 1, data_size,
1871 } else if (data_buf) {
1872 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1874 nandc_set_read_loc(nandc, 0, data_size,
1879 config_nand_cw_read(nandc, true);
1882 read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1886 * when ecc is enabled, the controller doesn't read the real
1887 * or dummy bad block markers in each chunk. To maintain a
1888 * consistent layout across RAW and ECC reads, we just
1889 * leave the real/dummy BBM offsets empty (i.e, filled with
1895 for (j = 0; j < host->bbm_size; j++)
1898 read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1899 oob_buf, oob_size, 0);
1903 data_buf += data_size;
1905 oob_buf += oob_size;
1908 ret = submit_descs(nandc);
1912 dev_err(nandc->dev, "failure to read page/oob\n");
1916 return parse_read_errors(host, data_buf_start, oob_buf_start, page);
1920 * a helper that copies the last step/codeword of a page (containing free oob)
1921 * into our local buffer
1923 static int copy_last_cw(struct qcom_nand_host *host, int page)
1925 struct nand_chip *chip = &host->chip;
1926 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1927 struct nand_ecc_ctrl *ecc = &chip->ecc;
1931 clear_read_regs(nandc);
1933 size = host->use_ecc ? host->cw_data : host->cw_size;
1935 /* prepare a clean read buffer */
1936 memset(nandc->data_buffer, 0xff, size);
1938 set_address(host, host->cw_size * (ecc->steps - 1), page);
1939 update_rw_regs(host, 1, true);
1941 config_nand_single_cw_page_read(nandc, host->use_ecc);
1943 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1945 ret = submit_descs(nandc);
1947 dev_err(nandc->dev, "failed to copy last codeword\n");
1954 /* implements ecc->read_page() */
1955 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1956 uint8_t *buf, int oob_required, int page)
1958 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1959 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1960 u8 *data_buf, *oob_buf = NULL;
1962 nand_read_page_op(chip, page, 0, NULL, 0);
1964 oob_buf = oob_required ? chip->oob_poi : NULL;
1966 clear_bam_transaction(nandc);
1968 return read_page_ecc(host, data_buf, oob_buf, page);
1971 /* implements ecc->read_page_raw() */
1972 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1973 struct nand_chip *chip, uint8_t *buf,
1974 int oob_required, int page)
1976 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1977 struct nand_ecc_ctrl *ecc = &chip->ecc;
1979 u8 *data_buf = buf, *oob_buf = chip->oob_poi;
1981 for (cw = 0; cw < ecc->steps; cw++) {
1982 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
1987 data_buf += host->cw_data;
1988 oob_buf += ecc->bytes;
1994 /* implements ecc->read_oob() */
1995 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1998 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1999 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2000 struct nand_ecc_ctrl *ecc = &chip->ecc;
2002 clear_read_regs(nandc);
2003 clear_bam_transaction(nandc);
2005 host->use_ecc = true;
2006 set_address(host, 0, page);
2007 update_rw_regs(host, ecc->steps, true);
2009 return read_page_ecc(host, NULL, chip->oob_poi, page);
2012 /* implements ecc->write_page() */
2013 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
2014 const uint8_t *buf, int oob_required, int page)
2016 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2017 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2018 struct nand_ecc_ctrl *ecc = &chip->ecc;
2019 u8 *data_buf, *oob_buf;
2022 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2024 clear_read_regs(nandc);
2025 clear_bam_transaction(nandc);
2027 data_buf = (u8 *)buf;
2028 oob_buf = chip->oob_poi;
2030 host->use_ecc = true;
2031 update_rw_regs(host, ecc->steps, false);
2032 config_nand_page_write(nandc);
2034 for (i = 0; i < ecc->steps; i++) {
2035 int data_size, oob_size;
2037 if (i == (ecc->steps - 1)) {
2038 data_size = ecc->size - ((ecc->steps - 1) << 2);
2039 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2042 data_size = host->cw_data;
2043 oob_size = ecc->bytes;
2047 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2048 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2051 * when ECC is enabled, we don't really need to write anything
2052 * to oob for the first n - 1 codewords since these oob regions
2053 * just contain ECC bytes that's written by the controller
2054 * itself. For the last codeword, we skip the bbm positions and
2055 * write to the free oob area.
2057 if (i == (ecc->steps - 1)) {
2058 oob_buf += host->bbm_size;
2060 write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2061 oob_buf, oob_size, 0);
2064 config_nand_cw_write(nandc);
2066 data_buf += data_size;
2067 oob_buf += oob_size;
2070 ret = submit_descs(nandc);
2072 dev_err(nandc->dev, "failure to write page\n");
2077 ret = nand_prog_page_end_op(chip);
2082 /* implements ecc->write_page_raw() */
2083 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
2084 struct nand_chip *chip, const uint8_t *buf,
2085 int oob_required, int page)
2087 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2088 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2089 struct nand_ecc_ctrl *ecc = &chip->ecc;
2090 u8 *data_buf, *oob_buf;
2093 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2094 clear_read_regs(nandc);
2095 clear_bam_transaction(nandc);
2097 data_buf = (u8 *)buf;
2098 oob_buf = chip->oob_poi;
2100 host->use_ecc = false;
2101 update_rw_regs(host, ecc->steps, false);
2102 config_nand_page_write(nandc);
2104 for (i = 0; i < ecc->steps; i++) {
2105 int data_size1, data_size2, oob_size1, oob_size2;
2106 int reg_off = FLASH_BUF_ACC;
2108 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2109 oob_size1 = host->bbm_size;
2111 if (i == (ecc->steps - 1)) {
2112 data_size2 = ecc->size - data_size1 -
2113 ((ecc->steps - 1) << 2);
2114 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2117 data_size2 = host->cw_data - data_size1;
2118 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2121 write_data_dma(nandc, reg_off, data_buf, data_size1,
2123 reg_off += data_size1;
2124 data_buf += data_size1;
2126 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2128 reg_off += oob_size1;
2129 oob_buf += oob_size1;
2131 write_data_dma(nandc, reg_off, data_buf, data_size2,
2133 reg_off += data_size2;
2134 data_buf += data_size2;
2136 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2137 oob_buf += oob_size2;
2139 config_nand_cw_write(nandc);
2142 ret = submit_descs(nandc);
2144 dev_err(nandc->dev, "failure to write raw page\n");
2149 ret = nand_prog_page_end_op(chip);
2155 * implements ecc->write_oob()
2157 * the NAND controller cannot write only data or only OOB within a codeword
2158 * since ECC is calculated for the combined codeword. So update the OOB from
2159 * chip->oob_poi, and pad the data area with OxFF before writing.
2161 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
2164 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2165 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2166 struct nand_ecc_ctrl *ecc = &chip->ecc;
2167 u8 *oob = chip->oob_poi;
2168 int data_size, oob_size;
2171 host->use_ecc = true;
2172 clear_bam_transaction(nandc);
2174 /* calculate the data and oob size for the last codeword/step */
2175 data_size = ecc->size - ((ecc->steps - 1) << 2);
2176 oob_size = mtd->oobavail;
2178 memset(nandc->data_buffer, 0xff, host->cw_data);
2179 /* override new oob content to last codeword */
2180 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2183 set_address(host, host->cw_size * (ecc->steps - 1), page);
2184 update_rw_regs(host, 1, false);
2186 config_nand_page_write(nandc);
2187 write_data_dma(nandc, FLASH_BUF_ACC,
2188 nandc->data_buffer, data_size + oob_size, 0);
2189 config_nand_cw_write(nandc);
2191 ret = submit_descs(nandc);
2196 dev_err(nandc->dev, "failure to write oob\n");
2200 return nand_prog_page_end_op(chip);
2203 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
2205 struct nand_chip *chip = mtd_to_nand(mtd);
2206 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2207 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2208 struct nand_ecc_ctrl *ecc = &chip->ecc;
2209 int page, ret, bbpos, bad = 0;
2211 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2214 * configure registers for a raw sub page read, the address is set to
2215 * the beginning of the last codeword, we don't care about reading ecc
2216 * portion of oob. we just want the first few bytes from this codeword
2217 * that contains the BBM
2219 host->use_ecc = false;
2221 clear_bam_transaction(nandc);
2222 ret = copy_last_cw(host, page);
2226 if (check_flash_errors(host, 1)) {
2227 dev_warn(nandc->dev, "error when trying to read BBM\n");
2231 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2233 bad = nandc->data_buffer[bbpos] != 0xff;
2235 if (chip->options & NAND_BUSWIDTH_16)
2236 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2241 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
2243 struct nand_chip *chip = mtd_to_nand(mtd);
2244 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2245 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2246 struct nand_ecc_ctrl *ecc = &chip->ecc;
2249 clear_read_regs(nandc);
2250 clear_bam_transaction(nandc);
2253 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2254 * we don't care about the rest of the content in the codeword since
2255 * we aren't going to use this block again
2257 memset(nandc->data_buffer, 0x00, host->cw_size);
2259 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2262 host->use_ecc = false;
2263 set_address(host, host->cw_size * (ecc->steps - 1), page);
2264 update_rw_regs(host, 1, false);
2266 config_nand_page_write(nandc);
2267 write_data_dma(nandc, FLASH_BUF_ACC,
2268 nandc->data_buffer, host->cw_size, 0);
2269 config_nand_cw_write(nandc);
2271 ret = submit_descs(nandc);
2276 dev_err(nandc->dev, "failure to update BBM\n");
2280 return nand_prog_page_end_op(chip);
2284 * the three functions below implement chip->read_byte(), chip->read_buf()
2285 * and chip->write_buf() respectively. these aren't used for
2286 * reading/writing page data, they are used for smaller data like reading
2289 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
2291 struct nand_chip *chip = mtd_to_nand(mtd);
2292 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2293 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2294 u8 *buf = nandc->data_buffer;
2297 if (host->last_command == NAND_CMD_STATUS) {
2300 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2305 if (nandc->buf_start < nandc->buf_count)
2306 ret = buf[nandc->buf_start++];
2311 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
2313 struct nand_chip *chip = mtd_to_nand(mtd);
2314 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2315 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2317 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2318 nandc->buf_start += real_len;
2321 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
2324 struct nand_chip *chip = mtd_to_nand(mtd);
2325 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2326 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2328 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2330 nandc->buf_start += real_len;
2333 /* we support only one external chip for now */
2334 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
2336 struct nand_chip *chip = mtd_to_nand(mtd);
2337 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2342 dev_warn(nandc->dev, "invalid chip select\n");
2346 * NAND controller page layout info
2348 * Layout with ECC enabled:
2350 * |----------------------| |---------------------------------|
2351 * | xx.......yy| | *********xx.......yy|
2352 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
2353 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
2354 * | xx.......yy| | *********xx.......yy|
2355 * |----------------------| |---------------------------------|
2356 * codeword 1,2..n-1 codeword n
2357 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
2359 * n = Number of codewords in the page
2361 * * = Spare/free bytes
2362 * x = Unused byte(s)
2363 * y = Reserved byte(s)
2365 * 2K page: n = 4, spare = 16 bytes
2366 * 4K page: n = 8, spare = 32 bytes
2367 * 8K page: n = 16, spare = 64 bytes
2369 * the qcom nand controller operates at a sub page/codeword level. each
2370 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2371 * the number of ECC bytes vary based on the ECC strength and the bus width.
2373 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2374 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2375 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2377 * When we access a page with ECC enabled, the reserved bytes(s) are not
2378 * accessible at all. When reading, we fill up these unreadable positions
2379 * with 0xffs. When writing, the controller skips writing the inaccessible
2382 * Layout with ECC disabled:
2384 * |------------------------------| |---------------------------------------|
2385 * | yy xx.......| | bb *********xx.......|
2386 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
2387 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
2388 * | yy xx.......| | bb *********xx.......|
2389 * |------------------------------| |---------------------------------------|
2390 * codeword 1,2..n-1 codeword n
2391 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
2393 * n = Number of codewords in the page
2395 * * = Spare/free bytes
2396 * x = Unused byte(s)
2397 * y = Dummy Bad Bock byte(s)
2398 * b = Real Bad Block byte(s)
2399 * size1/size2 = function of codeword size and 'n'
2401 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2402 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2403 * Block Markers. In the last codeword, this position contains the real BBM
2405 * In order to have a consistent layout between RAW and ECC modes, we assume
2406 * the following OOB layout arrangement:
2408 * |-----------| |--------------------|
2409 * |yyxx.......| |bb*********xx.......|
2410 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
2411 * |yyxx.......| |bb*********xx.......|
2412 * |yyxx.......| |bb*********xx.......|
2413 * |-----------| |--------------------|
2414 * first n - 1 nth OOB region
2417 * n = Number of codewords in the page
2419 * * = FREE OOB bytes
2420 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2421 * x = Unused byte(s)
2422 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2424 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2425 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2426 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2427 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2428 * the sum of the three).
2430 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2431 struct mtd_oob_region *oobregion)
2433 struct nand_chip *chip = mtd_to_nand(mtd);
2434 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2435 struct nand_ecc_ctrl *ecc = &chip->ecc;
2441 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2443 oobregion->offset = 0;
2445 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2446 oobregion->offset = mtd->oobsize - oobregion->length;
2452 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2453 struct mtd_oob_region *oobregion)
2455 struct nand_chip *chip = mtd_to_nand(mtd);
2456 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2457 struct nand_ecc_ctrl *ecc = &chip->ecc;
2462 oobregion->length = ecc->steps * 4;
2463 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2468 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2469 .ecc = qcom_nand_ooblayout_ecc,
2470 .free = qcom_nand_ooblayout_free,
2474 qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2476 return strength == 4 ? 12 : 16;
2478 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2479 NANDC_STEP_SIZE, 4, 8);
2481 static int qcom_nand_attach_chip(struct nand_chip *chip)
2483 struct mtd_info *mtd = nand_to_mtd(chip);
2484 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2485 struct nand_ecc_ctrl *ecc = &chip->ecc;
2486 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2487 int cwperpage, bad_block_byte, ret;
2491 /* controller only supports 512 bytes data steps */
2492 ecc->size = NANDC_STEP_SIZE;
2493 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2494 cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2497 * Each CW has 4 available OOB bytes which will be protected with ECC
2498 * so remaining bytes can be used for ECC.
2500 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2501 mtd->oobsize - (cwperpage * 4));
2503 dev_err(nandc->dev, "No valid ECC settings possible\n");
2507 if (ecc->strength >= 8) {
2508 /* 8 bit ECC defaults to BCH ECC on all platforms */
2509 host->bch_enabled = true;
2513 host->ecc_bytes_hw = 14;
2514 host->spare_bytes = 0;
2517 host->ecc_bytes_hw = 13;
2518 host->spare_bytes = 2;
2523 * if the controller supports BCH for 4 bit ECC, the controller
2524 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2527 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2529 host->bch_enabled = true;
2533 host->ecc_bytes_hw = 8;
2534 host->spare_bytes = 2;
2537 host->ecc_bytes_hw = 7;
2538 host->spare_bytes = 4;
2543 host->ecc_bytes_hw = 10;
2546 host->spare_bytes = 0;
2549 host->spare_bytes = 1;
2556 * we consider ecc->bytes as the sum of all the non-data content in a
2557 * step. It gives us a clean representation of the oob area (even if
2558 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2559 * ECC and 12 bytes for 4 bit ECC
2561 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2563 ecc->read_page = qcom_nandc_read_page;
2564 ecc->read_page_raw = qcom_nandc_read_page_raw;
2565 ecc->read_oob = qcom_nandc_read_oob;
2566 ecc->write_page = qcom_nandc_write_page;
2567 ecc->write_page_raw = qcom_nandc_write_page_raw;
2568 ecc->write_oob = qcom_nandc_write_oob;
2570 ecc->mode = NAND_ECC_HW;
2572 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2574 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2578 * DATA_UD_BYTES varies based on whether the read/write command protects
2579 * spare data with ECC too. We protect spare data by default, so we set
2580 * it to main + spare data, which are 512 and 4 bytes respectively.
2582 host->cw_data = 516;
2585 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2588 host->cw_size = host->cw_data + ecc->bytes;
2589 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2591 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2592 | host->cw_data << UD_SIZE_BYTES
2593 | 0 << DISABLE_STATUS_AFTER_WRITE
2594 | 5 << NUM_ADDR_CYCLES
2595 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2596 | 0 << STATUS_BFR_READ
2597 | 1 << SET_RD_MODE_AFTER_STATUS
2598 | host->spare_bytes << SPARE_SIZE_BYTES;
2600 host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2601 | 0 << CS_ACTIVE_BSY
2602 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2603 | 0 << BAD_BLOCK_IN_SPARE_AREA
2604 | 2 << WR_RD_BSY_GAP
2605 | wide_bus << WIDE_FLASH
2606 | host->bch_enabled << ENABLE_BCH_ECC;
2608 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2609 | host->cw_size << UD_SIZE_BYTES
2610 | 5 << NUM_ADDR_CYCLES
2611 | 0 << SPARE_SIZE_BYTES;
2613 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2614 | 0 << CS_ACTIVE_BSY
2615 | 17 << BAD_BLOCK_BYTE_NUM
2616 | 1 << BAD_BLOCK_IN_SPARE_AREA
2617 | 2 << WR_RD_BSY_GAP
2618 | wide_bus << WIDE_FLASH
2619 | 1 << DEV0_CFG1_ECC_DISABLE;
2621 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2623 | host->cw_data << ECC_NUM_DATA_BYTES
2624 | 1 << ECC_FORCE_CLK_OPEN
2625 | ecc_mode << ECC_MODE
2626 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2628 host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2630 host->clrflashstatus = FS_READY_BSY_N;
2631 host->clrreadstatus = 0xc0;
2632 nandc->regs->erased_cw_detect_cfg_clr =
2633 cpu_to_le32(CLR_ERASED_PAGE_DET);
2634 nandc->regs->erased_cw_detect_cfg_set =
2635 cpu_to_le32(SET_ERASED_PAGE_DET);
2638 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2639 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2640 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2646 static const struct nand_controller_ops qcom_nandc_ops = {
2647 .attach_chip = qcom_nand_attach_chip,
2650 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2654 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2656 dev_err(nandc->dev, "failed to set DMA mask\n");
2661 * we use the internal buffer for reading ONFI params, reading small
2662 * data like ID and status, and preforming read-copy-write operations
2663 * when writing to a codeword partially. 532 is the maximum possible
2664 * size of a codeword for our nand controller
2666 nandc->buf_size = 532;
2668 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2670 if (!nandc->data_buffer)
2673 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2678 nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2679 MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2681 if (!nandc->reg_read_buf)
2684 if (nandc->props->is_bam) {
2685 nandc->reg_read_dma =
2686 dma_map_single(nandc->dev, nandc->reg_read_buf,
2688 sizeof(*nandc->reg_read_buf),
2690 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2691 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2695 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2696 if (!nandc->tx_chan) {
2697 dev_err(nandc->dev, "failed to request tx channel\n");
2701 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2702 if (!nandc->rx_chan) {
2703 dev_err(nandc->dev, "failed to request rx channel\n");
2707 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2708 if (!nandc->cmd_chan) {
2709 dev_err(nandc->dev, "failed to request cmd channel\n");
2714 * Initially allocate BAM transaction to read ONFI param page.
2715 * After detecting all the devices, this BAM transaction will
2716 * be freed and the next BAM tranasction will be allocated with
2717 * maximum codeword size
2719 nandc->max_cwperpage = 1;
2720 nandc->bam_txn = alloc_bam_transaction(nandc);
2721 if (!nandc->bam_txn) {
2723 "failed to allocate bam transaction\n");
2727 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2730 "failed to request slave channel\n");
2735 INIT_LIST_HEAD(&nandc->desc_list);
2736 INIT_LIST_HEAD(&nandc->host_list);
2738 nand_controller_init(&nandc->controller);
2739 nandc->controller.ops = &qcom_nandc_ops;
2744 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2746 if (nandc->props->is_bam) {
2747 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2748 dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2750 sizeof(*nandc->reg_read_buf),
2754 dma_release_channel(nandc->tx_chan);
2757 dma_release_channel(nandc->rx_chan);
2759 if (nandc->cmd_chan)
2760 dma_release_channel(nandc->cmd_chan);
2763 dma_release_channel(nandc->chan);
2767 /* one time setup of a few nand controller registers */
2768 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2773 if (!nandc->props->is_qpic)
2774 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2775 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2776 NAND_DEV_CMD_VLD_VAL);
2778 /* enable ADM or BAM DMA */
2779 if (nandc->props->is_bam) {
2780 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2781 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2783 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2786 /* save the original values of these registers */
2787 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2788 nandc->vld = NAND_DEV_CMD_VLD_VAL;
2793 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2794 struct qcom_nand_host *host,
2795 struct device_node *dn)
2797 struct nand_chip *chip = &host->chip;
2798 struct mtd_info *mtd = nand_to_mtd(chip);
2799 struct device *dev = nandc->dev;
2802 ret = of_property_read_u32(dn, "reg", &host->cs);
2804 dev_err(dev, "can't get chip-select\n");
2808 nand_set_flash_node(chip, dn);
2809 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2813 mtd->owner = THIS_MODULE;
2814 mtd->dev.parent = dev;
2816 chip->cmdfunc = qcom_nandc_command;
2817 chip->select_chip = qcom_nandc_select_chip;
2818 chip->read_byte = qcom_nandc_read_byte;
2819 chip->read_buf = qcom_nandc_read_buf;
2820 chip->write_buf = qcom_nandc_write_buf;
2821 chip->set_features = nand_get_set_features_notsupp;
2822 chip->get_features = nand_get_set_features_notsupp;
2825 * the bad block marker is readable only when we read the last codeword
2826 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2827 * helpers don't allow us to read BB from a nand chip with ECC
2828 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2829 * and block_markbad helpers until we permanently switch to using
2830 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2832 chip->block_bad = qcom_nandc_block_bad;
2833 chip->block_markbad = qcom_nandc_block_markbad;
2835 chip->controller = &nandc->controller;
2836 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2839 /* set up initial status value */
2840 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2842 ret = nand_scan(chip, 1);
2846 if (nandc->props->is_bam) {
2847 free_bam_transaction(nandc);
2848 nandc->bam_txn = alloc_bam_transaction(nandc);
2849 if (!nandc->bam_txn) {
2851 "failed to allocate bam transaction\n");
2856 ret = mtd_device_register(mtd, NULL, 0);
2863 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2865 struct device *dev = nandc->dev;
2866 struct device_node *dn = dev->of_node, *child;
2867 struct qcom_nand_host *host;
2870 for_each_available_child_of_node(dn, child) {
2871 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2877 ret = qcom_nand_host_init_and_register(nandc, host, child);
2879 devm_kfree(dev, host);
2883 list_add_tail(&host->node, &nandc->host_list);
2889 /* parse custom DT properties here */
2890 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2892 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2893 struct device_node *np = nandc->dev->of_node;
2896 if (!nandc->props->is_bam) {
2897 ret = of_property_read_u32(np, "qcom,cmd-crci",
2900 dev_err(nandc->dev, "command CRCI unspecified\n");
2904 ret = of_property_read_u32(np, "qcom,data-crci",
2907 dev_err(nandc->dev, "data CRCI unspecified\n");
2915 static int qcom_nandc_probe(struct platform_device *pdev)
2917 struct qcom_nand_controller *nandc;
2918 const void *dev_data;
2919 struct device *dev = &pdev->dev;
2920 struct resource *res;
2923 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2927 platform_set_drvdata(pdev, nandc);
2930 dev_data = of_device_get_match_data(dev);
2932 dev_err(&pdev->dev, "failed to get device data\n");
2936 nandc->props = dev_data;
2938 nandc->core_clk = devm_clk_get(dev, "core");
2939 if (IS_ERR(nandc->core_clk))
2940 return PTR_ERR(nandc->core_clk);
2942 nandc->aon_clk = devm_clk_get(dev, "aon");
2943 if (IS_ERR(nandc->aon_clk))
2944 return PTR_ERR(nandc->aon_clk);
2946 ret = qcom_nandc_parse_dt(pdev);
2950 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2951 nandc->base = devm_ioremap_resource(dev, res);
2952 if (IS_ERR(nandc->base))
2953 return PTR_ERR(nandc->base);
2955 nandc->base_phys = res->start;
2956 nandc->base_dma = dma_map_resource(dev, res->start,
2958 DMA_BIDIRECTIONAL, 0);
2959 if (!nandc->base_dma)
2962 ret = qcom_nandc_alloc(nandc);
2964 goto err_nandc_alloc;
2966 ret = clk_prepare_enable(nandc->core_clk);
2970 ret = clk_prepare_enable(nandc->aon_clk);
2974 ret = qcom_nandc_setup(nandc);
2978 ret = qcom_probe_nand_devices(nandc);
2985 clk_disable_unprepare(nandc->aon_clk);
2987 clk_disable_unprepare(nandc->core_clk);
2989 qcom_nandc_unalloc(nandc);
2991 dma_unmap_resource(dev, res->start, resource_size(res),
2992 DMA_BIDIRECTIONAL, 0);
2997 static int qcom_nandc_remove(struct platform_device *pdev)
2999 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
3000 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3001 struct qcom_nand_host *host;
3003 list_for_each_entry(host, &nandc->host_list, node)
3004 nand_release(&host->chip);
3007 qcom_nandc_unalloc(nandc);
3009 clk_disable_unprepare(nandc->aon_clk);
3010 clk_disable_unprepare(nandc->core_clk);
3012 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3013 DMA_BIDIRECTIONAL, 0);
3018 static const struct qcom_nandc_props ipq806x_nandc_props = {
3019 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3021 .dev_cmd_reg_start = 0x0,
3024 static const struct qcom_nandc_props ipq4019_nandc_props = {
3025 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3028 .dev_cmd_reg_start = 0x0,
3031 static const struct qcom_nandc_props ipq8074_nandc_props = {
3032 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3035 .dev_cmd_reg_start = 0x7000,
3039 * data will hold a struct pointer containing more differences once we support
3040 * more controller variants
3042 static const struct of_device_id qcom_nandc_of_match[] = {
3044 .compatible = "qcom,ipq806x-nand",
3045 .data = &ipq806x_nandc_props,
3048 .compatible = "qcom,ipq4019-nand",
3049 .data = &ipq4019_nandc_props,
3052 .compatible = "qcom,ipq8074-nand",
3053 .data = &ipq8074_nandc_props,
3057 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3059 static struct platform_driver qcom_nandc_driver = {
3061 .name = "qcom-nandc",
3062 .of_match_table = qcom_nandc_of_match,
3064 .probe = qcom_nandc_probe,
3065 .remove = qcom_nandc_remove,
3067 module_platform_driver(qcom_nandc_driver);
3069 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3070 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3071 MODULE_LICENSE("GPL v2");