2 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 #include <linux/clk.h>
14 #include <linux/slab.h>
15 #include <linux/bitops.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmaengine.h>
18 #include <linux/module.h>
19 #include <linux/mtd/rawnand.h>
20 #include <linux/mtd/partitions.h>
22 #include <linux/of_device.h>
23 #include <linux/delay.h>
24 #include <linux/dma/qcom_bam_dma.h>
26 /* NANDc reg offsets */
27 #define NAND_FLASH_CMD 0x00
28 #define NAND_ADDR0 0x04
29 #define NAND_ADDR1 0x08
30 #define NAND_FLASH_CHIP_SELECT 0x0c
31 #define NAND_EXEC_CMD 0x10
32 #define NAND_FLASH_STATUS 0x14
33 #define NAND_BUFFER_STATUS 0x18
34 #define NAND_DEV0_CFG0 0x20
35 #define NAND_DEV0_CFG1 0x24
36 #define NAND_DEV0_ECC_CFG 0x28
37 #define NAND_DEV1_ECC_CFG 0x2c
38 #define NAND_DEV1_CFG0 0x30
39 #define NAND_DEV1_CFG1 0x34
40 #define NAND_READ_ID 0x40
41 #define NAND_READ_STATUS 0x44
42 #define NAND_DEV_CMD0 0xa0
43 #define NAND_DEV_CMD1 0xa4
44 #define NAND_DEV_CMD2 0xa8
45 #define NAND_DEV_CMD_VLD 0xac
46 #define SFLASHC_BURST_CFG 0xe0
47 #define NAND_ERASED_CW_DETECT_CFG 0xe8
48 #define NAND_ERASED_CW_DETECT_STATUS 0xec
49 #define NAND_EBI2_ECC_BUF_CFG 0xf0
50 #define FLASH_BUF_ACC 0x100
52 #define NAND_CTRL 0xf00
53 #define NAND_VERSION 0xf08
54 #define NAND_READ_LOCATION_0 0xf20
55 #define NAND_READ_LOCATION_1 0xf24
56 #define NAND_READ_LOCATION_2 0xf28
57 #define NAND_READ_LOCATION_3 0xf2c
59 /* dummy register offsets, used by write_reg_dma */
60 #define NAND_DEV_CMD1_RESTORE 0xdead
61 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef
63 /* NAND_FLASH_CMD bits */
64 #define PAGE_ACC BIT(4)
65 #define LAST_PAGE BIT(5)
67 /* NAND_FLASH_CHIP_SELECT bits */
68 #define NAND_DEV_SEL 0
71 /* NAND_FLASH_STATUS bits */
72 #define FS_OP_ERR BIT(4)
73 #define FS_READY_BSY_N BIT(5)
74 #define FS_MPU_ERR BIT(8)
75 #define FS_DEVICE_STS_ERR BIT(16)
76 #define FS_DEVICE_WP BIT(23)
78 /* NAND_BUFFER_STATUS bits */
79 #define BS_UNCORRECTABLE_BIT BIT(8)
80 #define BS_CORRECTABLE_ERR_MSK 0x1f
82 /* NAND_DEVn_CFG0 bits */
83 #define DISABLE_STATUS_AFTER_WRITE 4
85 #define UD_SIZE_BYTES 9
86 #define ECC_PARITY_SIZE_BYTES_RS 19
87 #define SPARE_SIZE_BYTES 23
88 #define NUM_ADDR_CYCLES 27
89 #define STATUS_BFR_READ 30
90 #define SET_RD_MODE_AFTER_STATUS 31
92 /* NAND_DEVn_CFG0 bits */
93 #define DEV0_CFG1_ECC_DISABLE 0
95 #define NAND_RECOVERY_CYCLES 2
96 #define CS_ACTIVE_BSY 5
97 #define BAD_BLOCK_BYTE_NUM 6
98 #define BAD_BLOCK_IN_SPARE_AREA 16
99 #define WR_RD_BSY_GAP 17
100 #define ENABLE_BCH_ECC 27
102 /* NAND_DEV0_ECC_CFG bits */
103 #define ECC_CFG_ECC_DISABLE 0
104 #define ECC_SW_RESET 1
106 #define ECC_PARITY_SIZE_BYTES_BCH 8
107 #define ECC_NUM_DATA_BYTES 16
108 #define ECC_FORCE_CLK_OPEN 30
110 /* NAND_DEV_CMD1 bits */
113 /* NAND_DEV_CMD_VLD bits */
114 #define READ_START_VLD BIT(0)
115 #define READ_STOP_VLD BIT(1)
116 #define WRITE_START_VLD BIT(2)
117 #define ERASE_START_VLD BIT(3)
118 #define SEQ_READ_START_VLD BIT(4)
120 /* NAND_EBI2_ECC_BUF_CFG bits */
123 /* NAND_ERASED_CW_DETECT_CFG bits */
124 #define ERASED_CW_ECC_MASK 1
125 #define AUTO_DETECT_RES 0
126 #define MASK_ECC (1 << ERASED_CW_ECC_MASK)
127 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
128 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
129 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
130 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
132 /* NAND_ERASED_CW_DETECT_STATUS bits */
133 #define PAGE_ALL_ERASED BIT(7)
134 #define CODEWORD_ALL_ERASED BIT(6)
135 #define PAGE_ERASED BIT(5)
136 #define CODEWORD_ERASED BIT(4)
137 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
138 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
140 /* NAND_READ_LOCATION_n bits */
141 #define READ_LOCATION_OFFSET 0
142 #define READ_LOCATION_SIZE 16
143 #define READ_LOCATION_LAST 31
146 #define NAND_VERSION_MAJOR_MASK 0xf0000000
147 #define NAND_VERSION_MAJOR_SHIFT 28
148 #define NAND_VERSION_MINOR_MASK 0x0fff0000
149 #define NAND_VERSION_MINOR_SHIFT 16
152 #define OP_PAGE_READ 0x2
153 #define OP_PAGE_READ_WITH_ECC 0x3
154 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4
155 #define OP_PROGRAM_PAGE 0x6
156 #define OP_PAGE_PROGRAM_WITH_ECC 0x7
157 #define OP_PROGRAM_PAGE_SPARE 0x9
158 #define OP_BLOCK_ERASE 0xa
159 #define OP_FETCH_ID 0xb
160 #define OP_RESET_DEVICE 0xd
162 /* Default Value for NAND_DEV_CMD_VLD */
163 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \
164 ERASE_START_VLD | SEQ_READ_START_VLD)
167 #define BAM_MODE_EN BIT(0)
170 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
171 * the driver calls the chunks 'step' or 'codeword' interchangeably
173 #define NANDC_STEP_SIZE 512
176 * the largest page size we support is 8K, this will have 16 steps/codewords
179 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
181 /* we read at most 3 registers per codeword scan */
182 #define MAX_REG_RD (3 * MAX_NUM_STEPS)
184 /* ECC modes supported by the controller */
185 #define ECC_NONE BIT(0)
186 #define ECC_RS_4BIT BIT(1)
187 #define ECC_BCH_4BIT BIT(2)
188 #define ECC_BCH_8BIT BIT(3)
190 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \
191 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \
192 ((offset) << READ_LOCATION_OFFSET) | \
193 ((size) << READ_LOCATION_SIZE) | \
194 ((is_last) << READ_LOCATION_LAST))
197 * Returns the actual register address for all NAND_DEV_ registers
198 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
200 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
202 /* Returns the NAND register physical address */
203 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
205 /* Returns the dma address for reg read buffer */
206 #define reg_buf_dma_addr(chip, vaddr) \
207 ((chip)->reg_read_dma + \
208 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
210 #define QPIC_PER_CW_CMD_ELEMENTS 32
211 #define QPIC_PER_CW_CMD_SGL 32
212 #define QPIC_PER_CW_DATA_SGL 8
214 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000)
217 * Flags used in DMA descriptor preparation helper functions
218 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
220 /* Don't set the EOT in current tx BAM sgl */
221 #define NAND_BAM_NO_EOT BIT(0)
222 /* Set the NWD flag in current BAM sgl */
223 #define NAND_BAM_NWD BIT(1)
224 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
225 #define NAND_BAM_NEXT_SGL BIT(2)
227 * Erased codeword status is being used two times in single transfer so this
228 * flag will determine the current value of erased codeword status register
230 #define NAND_ERASED_CW_SET BIT(4)
233 * This data type corresponds to the BAM transaction which will be used for all
235 * @bam_ce - the array of BAM command elements
236 * @cmd_sgl - sgl for NAND BAM command pipe
237 * @data_sgl - sgl for NAND BAM consumer/producer pipe
238 * @bam_ce_pos - the index in bam_ce which is available for next sgl
239 * @bam_ce_start - the index in bam_ce which marks the start position ce
240 * for current sgl. It will be used for size calculation
242 * @cmd_sgl_pos - current index in command sgl.
243 * @cmd_sgl_start - start index in command sgl.
244 * @tx_sgl_pos - current index in data sgl for tx.
245 * @tx_sgl_start - start index in data sgl for tx.
246 * @rx_sgl_pos - current index in data sgl for rx.
247 * @rx_sgl_start - start index in data sgl for rx.
248 * @wait_second_completion - wait for second DMA desc completion before making
249 * the NAND transfer completion.
250 * @txn_done - completion for NAND transfer.
251 * @last_data_desc - last DMA desc in data channel (tx/rx).
252 * @last_cmd_desc - last DMA desc in command channel.
254 struct bam_transaction {
255 struct bam_cmd_element *bam_ce;
256 struct scatterlist *cmd_sgl;
257 struct scatterlist *data_sgl;
266 bool wait_second_completion;
267 struct completion txn_done;
268 struct dma_async_tx_descriptor *last_data_desc;
269 struct dma_async_tx_descriptor *last_cmd_desc;
273 * This data type corresponds to the nand dma descriptor
274 * @list - list for desc_info
275 * @dir - DMA transfer direction
276 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
278 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
279 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
280 * @dma_desc - low level DMA engine descriptor
283 struct list_head node;
285 enum dma_data_direction dir;
287 struct scatterlist adm_sgl;
289 struct scatterlist *bam_sgl;
293 struct dma_async_tx_descriptor *dma_desc;
297 * holds the current register values that we want to write. acts as a contiguous
298 * chunk of memory which we use to write the controller registers through DMA.
311 __le32 clrflashstatus;
312 __le32 clrreadstatus;
321 __le32 read_location0;
322 __le32 read_location1;
323 __le32 read_location2;
324 __le32 read_location3;
326 __le32 erased_cw_detect_cfg_clr;
327 __le32 erased_cw_detect_cfg_set;
331 * NAND controller data struct
333 * @controller: base controller structure
334 * @host_list: list containing all the chips attached to the
336 * @dev: parent device
338 * @base_phys: physical base address of controller registers
339 * @base_dma: dma base address of controller registers
340 * @core_clk: controller clock
341 * @aon_clk: another controller clock
344 * @cmd_crci: ADM DMA CRCI for command flow control
345 * @data_crci: ADM DMA CRCI for data flow control
346 * @desc_list: DMA descriptor list (list of desc_infos)
348 * @data_buffer: our local DMA buffer for page read/writes,
349 * used when we can't use the buffer provided
350 * by upper layers directly
351 * @buf_size/count/start: markers for chip->read_buf/write_buf functions
352 * @reg_read_buf: local buffer for reading back registers via DMA
353 * @reg_read_dma: contains dma address for register read buffer
354 * @reg_read_pos: marker for data read in reg_read_buf
356 * @regs: a contiguous chunk of memory for DMA register
357 * writes. contains the register values to be
358 * written to controller
359 * @cmd1/vld: some fixed controller register values
360 * @props: properties of current NAND controller,
361 * initialized via DT match data
362 * @max_cwperpage: maximum QPIC codewords required. calculated
363 * from all connected NAND devices pagesize
365 struct qcom_nand_controller {
366 struct nand_controller controller;
367 struct list_head host_list;
372 phys_addr_t base_phys;
375 struct clk *core_clk;
379 /* will be used only by QPIC for BAM DMA */
381 struct dma_chan *tx_chan;
382 struct dma_chan *rx_chan;
383 struct dma_chan *cmd_chan;
386 /* will be used only by EBI2 for ADM DMA */
388 struct dma_chan *chan;
389 unsigned int cmd_crci;
390 unsigned int data_crci;
394 struct list_head desc_list;
395 struct bam_transaction *bam_txn;
401 unsigned int max_cwperpage;
403 __le32 *reg_read_buf;
404 dma_addr_t reg_read_dma;
407 struct nandc_regs *regs;
410 const struct qcom_nandc_props *props;
414 * NAND chip structure
416 * @chip: base NAND chip structure
417 * @node: list node to add itself to host_list in
418 * qcom_nand_controller
420 * @cs: chip select value for this chip
421 * @cw_size: the number of bytes in a single step/codeword
422 * of a page, consisting of all data, ecc, spare
424 * @cw_data: the number of bytes within a codeword protected
426 * @use_ecc: request the controller to use ECC for the
427 * upcoming read/write
428 * @bch_enabled: flag to tell whether BCH ECC mode is used
429 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
431 * @status: value to be returned if NAND_CMD_STATUS command
433 * @last_command: keeps track of last command on this chip. used
434 * for reading correct status
436 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
437 * ecc/non-ecc mode for the current nand flash
440 struct qcom_nand_host {
441 struct nand_chip chip;
442 struct list_head node;
456 u32 cfg0_raw, cfg1_raw;
464 * This data type corresponds to the NAND controller properties which varies
465 * among different NAND controllers.
466 * @ecc_modes - ecc mode for NAND
467 * @is_bam - whether NAND controller is using BAM
468 * @is_qpic - whether NAND CTRL is part of qpic IP
469 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
471 struct qcom_nandc_props {
475 u32 dev_cmd_reg_start;
478 /* Frees the BAM transaction memory */
479 static void free_bam_transaction(struct qcom_nand_controller *nandc)
481 struct bam_transaction *bam_txn = nandc->bam_txn;
483 devm_kfree(nandc->dev, bam_txn);
486 /* Allocates and Initializes the BAM transaction */
487 static struct bam_transaction *
488 alloc_bam_transaction(struct qcom_nand_controller *nandc)
490 struct bam_transaction *bam_txn;
492 unsigned int num_cw = nandc->max_cwperpage;
496 sizeof(*bam_txn) + num_cw *
497 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
498 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
499 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
501 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
505 bam_txn = bam_txn_buf;
506 bam_txn_buf += sizeof(*bam_txn);
508 bam_txn->bam_ce = bam_txn_buf;
510 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
512 bam_txn->cmd_sgl = bam_txn_buf;
514 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
516 bam_txn->data_sgl = bam_txn_buf;
518 init_completion(&bam_txn->txn_done);
523 /* Clears the BAM transaction indexes */
524 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
526 struct bam_transaction *bam_txn = nandc->bam_txn;
528 if (!nandc->props->is_bam)
531 bam_txn->bam_ce_pos = 0;
532 bam_txn->bam_ce_start = 0;
533 bam_txn->cmd_sgl_pos = 0;
534 bam_txn->cmd_sgl_start = 0;
535 bam_txn->tx_sgl_pos = 0;
536 bam_txn->tx_sgl_start = 0;
537 bam_txn->rx_sgl_pos = 0;
538 bam_txn->rx_sgl_start = 0;
539 bam_txn->last_data_desc = NULL;
540 bam_txn->wait_second_completion = false;
542 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
543 QPIC_PER_CW_CMD_SGL);
544 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
545 QPIC_PER_CW_DATA_SGL);
547 reinit_completion(&bam_txn->txn_done);
550 /* Callback for DMA descriptor completion */
551 static void qpic_bam_dma_done(void *data)
553 struct bam_transaction *bam_txn = data;
556 * In case of data transfer with NAND, 2 callbacks will be generated.
557 * One for command channel and another one for data channel.
558 * If current transaction has data descriptors
559 * (i.e. wait_second_completion is true), then set this to false
560 * and wait for second DMA descriptor completion.
562 if (bam_txn->wait_second_completion)
563 bam_txn->wait_second_completion = false;
565 complete(&bam_txn->txn_done);
568 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
570 return container_of(chip, struct qcom_nand_host, chip);
573 static inline struct qcom_nand_controller *
574 get_qcom_nand_controller(struct nand_chip *chip)
576 return container_of(chip->controller, struct qcom_nand_controller,
580 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
582 return ioread32(nandc->base + offset);
585 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
588 iowrite32(val, nandc->base + offset);
591 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
594 if (!nandc->props->is_bam)
598 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
600 sizeof(*nandc->reg_read_buf),
603 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
605 sizeof(*nandc->reg_read_buf),
609 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
618 case NAND_FLASH_CHIP_SELECT:
619 return ®s->chip_sel;
622 case NAND_FLASH_STATUS:
623 return ®s->clrflashstatus;
628 case NAND_DEV0_ECC_CFG:
629 return ®s->ecc_bch_cfg;
630 case NAND_READ_STATUS:
631 return ®s->clrreadstatus;
634 case NAND_DEV_CMD1_RESTORE:
635 return ®s->orig_cmd1;
636 case NAND_DEV_CMD_VLD:
638 case NAND_DEV_CMD_VLD_RESTORE:
639 return ®s->orig_vld;
640 case NAND_EBI2_ECC_BUF_CFG:
641 return ®s->ecc_buf_cfg;
642 case NAND_READ_LOCATION_0:
643 return ®s->read_location0;
644 case NAND_READ_LOCATION_1:
645 return ®s->read_location1;
646 case NAND_READ_LOCATION_2:
647 return ®s->read_location2;
648 case NAND_READ_LOCATION_3:
649 return ®s->read_location3;
655 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
658 struct nandc_regs *regs = nandc->regs;
661 reg = offset_to_nandc_reg(regs, offset);
664 *reg = cpu_to_le32(val);
667 /* helper to configure address register values */
668 static void set_address(struct qcom_nand_host *host, u16 column, int page)
670 struct nand_chip *chip = &host->chip;
671 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
673 if (chip->options & NAND_BUSWIDTH_16)
676 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
677 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
681 * update_rw_regs: set up read/write register values, these will be
682 * written to the NAND controller registers via DMA
684 * @num_cw: number of steps for the read/write operation
685 * @read: read or write operation
687 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
689 struct nand_chip *chip = &host->chip;
690 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
691 u32 cmd, cfg0, cfg1, ecc_bch_cfg;
695 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
697 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
699 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
703 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
704 (num_cw - 1) << CW_PER_PAGE;
707 ecc_bch_cfg = host->ecc_bch_cfg;
709 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
710 (num_cw - 1) << CW_PER_PAGE;
712 cfg1 = host->cfg1_raw;
713 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
716 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
717 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
718 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
719 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
720 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
721 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
722 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
723 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
726 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
727 host->cw_data : host->cw_size, 1);
731 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
732 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
733 * which will be submitted to DMA engine.
735 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
736 struct dma_chan *chan,
739 struct desc_info *desc;
740 struct scatterlist *sgl;
741 unsigned int sgl_cnt;
743 struct bam_transaction *bam_txn = nandc->bam_txn;
744 enum dma_transfer_direction dir_eng;
745 struct dma_async_tx_descriptor *dma_desc;
747 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
751 if (chan == nandc->cmd_chan) {
752 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
753 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
754 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
755 dir_eng = DMA_MEM_TO_DEV;
756 desc->dir = DMA_TO_DEVICE;
757 } else if (chan == nandc->tx_chan) {
758 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
759 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
760 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
761 dir_eng = DMA_MEM_TO_DEV;
762 desc->dir = DMA_TO_DEVICE;
764 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
765 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
766 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
767 dir_eng = DMA_DEV_TO_MEM;
768 desc->dir = DMA_FROM_DEVICE;
771 sg_mark_end(sgl + sgl_cnt - 1);
772 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
774 dev_err(nandc->dev, "failure in mapping desc\n");
779 desc->sgl_cnt = sgl_cnt;
782 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
786 dev_err(nandc->dev, "failure in prep desc\n");
787 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
792 desc->dma_desc = dma_desc;
794 /* update last data/command descriptor */
795 if (chan == nandc->cmd_chan)
796 bam_txn->last_cmd_desc = dma_desc;
798 bam_txn->last_data_desc = dma_desc;
800 list_add_tail(&desc->node, &nandc->desc_list);
806 * Prepares the command descriptor for BAM DMA which will be used for NAND
807 * register reads and writes. The command descriptor requires the command
808 * to be formed in command element type so this function uses the command
809 * element from bam transaction ce array and fills the same with required
810 * data. A single SGL can contain multiple command elements so
811 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
812 * after the current command element.
814 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
815 int reg_off, const void *vaddr,
816 int size, unsigned int flags)
820 struct bam_cmd_element *bam_ce_buffer;
821 struct bam_transaction *bam_txn = nandc->bam_txn;
823 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
825 /* fill the command desc */
826 for (i = 0; i < size; i++) {
828 bam_prep_ce(&bam_ce_buffer[i],
829 nandc_reg_phys(nandc, reg_off + 4 * i),
831 reg_buf_dma_addr(nandc,
832 (__le32 *)vaddr + i));
834 bam_prep_ce_le32(&bam_ce_buffer[i],
835 nandc_reg_phys(nandc, reg_off + 4 * i),
837 *((__le32 *)vaddr + i));
840 bam_txn->bam_ce_pos += size;
842 /* use the separate sgl after this command */
843 if (flags & NAND_BAM_NEXT_SGL) {
844 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
845 bam_ce_size = (bam_txn->bam_ce_pos -
846 bam_txn->bam_ce_start) *
847 sizeof(struct bam_cmd_element);
848 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
849 bam_ce_buffer, bam_ce_size);
850 bam_txn->cmd_sgl_pos++;
851 bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
853 if (flags & NAND_BAM_NWD) {
854 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
866 * Prepares the data descriptor for BAM DMA which will be used for NAND
867 * data reads and writes.
869 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
871 int size, unsigned int flags)
874 struct bam_transaction *bam_txn = nandc->bam_txn;
877 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
879 bam_txn->rx_sgl_pos++;
881 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
883 bam_txn->tx_sgl_pos++;
886 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
887 * is not set, form the DMA descriptor
889 if (!(flags & NAND_BAM_NO_EOT)) {
890 ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
900 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
901 int reg_off, const void *vaddr, int size,
904 struct desc_info *desc;
905 struct dma_async_tx_descriptor *dma_desc;
906 struct scatterlist *sgl;
907 struct dma_slave_config slave_conf;
908 enum dma_transfer_direction dir_eng;
911 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
915 sgl = &desc->adm_sgl;
917 sg_init_one(sgl, vaddr, size);
920 dir_eng = DMA_DEV_TO_MEM;
921 desc->dir = DMA_FROM_DEVICE;
923 dir_eng = DMA_MEM_TO_DEV;
924 desc->dir = DMA_TO_DEVICE;
927 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
933 memset(&slave_conf, 0x00, sizeof(slave_conf));
935 slave_conf.device_fc = flow_control;
937 slave_conf.src_maxburst = 16;
938 slave_conf.src_addr = nandc->base_dma + reg_off;
939 slave_conf.slave_id = nandc->data_crci;
941 slave_conf.dst_maxburst = 16;
942 slave_conf.dst_addr = nandc->base_dma + reg_off;
943 slave_conf.slave_id = nandc->cmd_crci;
946 ret = dmaengine_slave_config(nandc->chan, &slave_conf);
948 dev_err(nandc->dev, "failed to configure dma channel\n");
952 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
954 dev_err(nandc->dev, "failed to prepare desc\n");
959 desc->dma_desc = dma_desc;
961 list_add_tail(&desc->node, &nandc->desc_list);
971 * read_reg_dma: prepares a descriptor to read a given number of
972 * contiguous registers to the reg_read_buf pointer
974 * @first: offset of the first register in the contiguous block
975 * @num_regs: number of registers to read
976 * @flags: flags to control DMA descriptor preparation
978 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
979 int num_regs, unsigned int flags)
981 bool flow_control = false;
984 vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
985 nandc->reg_read_pos += num_regs;
987 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
988 first = dev_cmd_reg_addr(nandc, first);
990 if (nandc->props->is_bam)
991 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
994 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
997 return prep_adm_dma_desc(nandc, true, first, vaddr,
998 num_regs * sizeof(u32), flow_control);
1002 * write_reg_dma: prepares a descriptor to write a given number of
1003 * contiguous registers
1005 * @first: offset of the first register in the contiguous block
1006 * @num_regs: number of registers to write
1007 * @flags: flags to control DMA descriptor preparation
1009 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1010 int num_regs, unsigned int flags)
1012 bool flow_control = false;
1013 struct nandc_regs *regs = nandc->regs;
1016 vaddr = offset_to_nandc_reg(regs, first);
1018 if (first == NAND_ERASED_CW_DETECT_CFG) {
1019 if (flags & NAND_ERASED_CW_SET)
1020 vaddr = ®s->erased_cw_detect_cfg_set;
1022 vaddr = ®s->erased_cw_detect_cfg_clr;
1025 if (first == NAND_EXEC_CMD)
1026 flags |= NAND_BAM_NWD;
1028 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1029 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1031 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1032 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1034 if (nandc->props->is_bam)
1035 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1038 if (first == NAND_FLASH_CMD)
1039 flow_control = true;
1041 return prep_adm_dma_desc(nandc, false, first, vaddr,
1042 num_regs * sizeof(u32), flow_control);
1046 * read_data_dma: prepares a DMA descriptor to transfer data from the
1047 * controller's internal buffer to the buffer 'vaddr'
1049 * @reg_off: offset within the controller's data buffer
1050 * @vaddr: virtual address of the buffer we want to write to
1051 * @size: DMA transaction size in bytes
1052 * @flags: flags to control DMA descriptor preparation
1054 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1055 const u8 *vaddr, int size, unsigned int flags)
1057 if (nandc->props->is_bam)
1058 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1060 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1064 * write_data_dma: prepares a DMA descriptor to transfer data from
1065 * 'vaddr' to the controller's internal buffer
1067 * @reg_off: offset within the controller's data buffer
1068 * @vaddr: virtual address of the buffer we want to read from
1069 * @size: DMA transaction size in bytes
1070 * @flags: flags to control DMA descriptor preparation
1072 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1073 const u8 *vaddr, int size, unsigned int flags)
1075 if (nandc->props->is_bam)
1076 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1078 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1082 * Helper to prepare DMA descriptors for configuring registers
1083 * before reading a NAND page.
1085 static void config_nand_page_read(struct qcom_nand_controller *nandc)
1087 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1088 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1089 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1090 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1091 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1092 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1096 * Helper to prepare DMA descriptors for configuring registers
1097 * before reading each codeword in NAND page.
1100 config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc)
1102 if (nandc->props->is_bam)
1103 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
1106 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1107 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1110 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1111 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1114 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1119 * Helper to prepare dma descriptors to configure registers needed for reading a
1120 * single codeword in page
1123 config_nand_single_cw_page_read(struct qcom_nand_controller *nandc,
1126 config_nand_page_read(nandc);
1127 config_nand_cw_read(nandc, use_ecc);
1131 * Helper to prepare DMA descriptors used to configure registers needed for
1132 * before writing a NAND page.
1134 static void config_nand_page_write(struct qcom_nand_controller *nandc)
1136 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1137 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1138 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1143 * Helper to prepare DMA descriptors for configuring registers
1144 * before writing each codeword in NAND page.
1146 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1148 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1149 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1151 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1153 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1154 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1158 * the following functions are used within chip->cmdfunc() to perform different
1159 * NAND_CMD_* commands
1162 /* sets up descriptors for NAND_CMD_PARAM */
1163 static int nandc_param(struct qcom_nand_host *host)
1165 struct nand_chip *chip = &host->chip;
1166 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1169 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1170 * in use. we configure the controller to perform a raw read of 512
1171 * bytes to read onfi params
1173 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1174 nandc_set_reg(nandc, NAND_ADDR0, 0);
1175 nandc_set_reg(nandc, NAND_ADDR1, 0);
1176 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1177 | 512 << UD_SIZE_BYTES
1178 | 5 << NUM_ADDR_CYCLES
1179 | 0 << SPARE_SIZE_BYTES);
1180 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1181 | 0 << CS_ACTIVE_BSY
1182 | 17 << BAD_BLOCK_BYTE_NUM
1183 | 1 << BAD_BLOCK_IN_SPARE_AREA
1184 | 2 << WR_RD_BSY_GAP
1186 | 1 << DEV0_CFG1_ECC_DISABLE);
1187 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1189 /* configure CMD1 and VLD for ONFI param probing */
1190 nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1191 (nandc->vld & ~READ_START_VLD));
1192 nandc_set_reg(nandc, NAND_DEV_CMD1,
1193 (nandc->cmd1 & ~(0xFF << READ_ADDR))
1194 | NAND_CMD_PARAM << READ_ADDR);
1196 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1198 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1199 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1200 nandc_set_read_loc(nandc, 0, 0, 512, 1);
1202 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1203 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1205 nandc->buf_count = 512;
1206 memset(nandc->data_buffer, 0xff, nandc->buf_count);
1208 config_nand_single_cw_page_read(nandc, false);
1210 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1211 nandc->buf_count, 0);
1213 /* restore CMD1 and VLD regs */
1214 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1215 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1220 /* sets up descriptors for NAND_CMD_ERASE1 */
1221 static int erase_block(struct qcom_nand_host *host, int page_addr)
1223 struct nand_chip *chip = &host->chip;
1224 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1226 nandc_set_reg(nandc, NAND_FLASH_CMD,
1227 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1228 nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1229 nandc_set_reg(nandc, NAND_ADDR1, 0);
1230 nandc_set_reg(nandc, NAND_DEV0_CFG0,
1231 host->cfg0_raw & ~(7 << CW_PER_PAGE));
1232 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1233 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1234 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1235 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1237 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1238 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1239 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1241 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1243 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1244 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1249 /* sets up descriptors for NAND_CMD_READID */
1250 static int read_id(struct qcom_nand_host *host, int column)
1252 struct nand_chip *chip = &host->chip;
1253 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1258 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1259 nandc_set_reg(nandc, NAND_ADDR0, column);
1260 nandc_set_reg(nandc, NAND_ADDR1, 0);
1261 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1262 nandc->props->is_bam ? 0 : DM_EN);
1263 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1265 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1266 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1268 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1273 /* sets up descriptors for NAND_CMD_RESET */
1274 static int reset(struct qcom_nand_host *host)
1276 struct nand_chip *chip = &host->chip;
1277 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1279 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1280 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1282 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1283 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1285 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1290 /* helpers to submit/free our list of dma descriptors */
1291 static int submit_descs(struct qcom_nand_controller *nandc)
1293 struct desc_info *desc;
1294 dma_cookie_t cookie = 0;
1295 struct bam_transaction *bam_txn = nandc->bam_txn;
1298 if (nandc->props->is_bam) {
1299 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1300 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1305 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1306 r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1307 DMA_PREP_INTERRUPT);
1312 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1313 r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1320 list_for_each_entry(desc, &nandc->desc_list, node)
1321 cookie = dmaengine_submit(desc->dma_desc);
1323 if (nandc->props->is_bam) {
1324 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1325 bam_txn->last_cmd_desc->callback_param = bam_txn;
1326 if (bam_txn->last_data_desc) {
1327 bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1328 bam_txn->last_data_desc->callback_param = bam_txn;
1329 bam_txn->wait_second_completion = true;
1332 dma_async_issue_pending(nandc->tx_chan);
1333 dma_async_issue_pending(nandc->rx_chan);
1334 dma_async_issue_pending(nandc->cmd_chan);
1336 if (!wait_for_completion_timeout(&bam_txn->txn_done,
1337 QPIC_NAND_COMPLETION_TIMEOUT))
1340 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1347 static void free_descs(struct qcom_nand_controller *nandc)
1349 struct desc_info *desc, *n;
1351 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1352 list_del(&desc->node);
1354 if (nandc->props->is_bam)
1355 dma_unmap_sg(nandc->dev, desc->bam_sgl,
1356 desc->sgl_cnt, desc->dir);
1358 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1365 /* reset the register read buffer for next NAND operation */
1366 static void clear_read_regs(struct qcom_nand_controller *nandc)
1368 nandc->reg_read_pos = 0;
1369 nandc_read_buffer_sync(nandc, false);
1372 static void pre_command(struct qcom_nand_host *host, int command)
1374 struct nand_chip *chip = &host->chip;
1375 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1377 nandc->buf_count = 0;
1378 nandc->buf_start = 0;
1379 host->use_ecc = false;
1380 host->last_command = command;
1382 clear_read_regs(nandc);
1384 if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1385 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1386 clear_bam_transaction(nandc);
1390 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1391 * privately maintained status byte, this status byte can be read after
1392 * NAND_CMD_STATUS is called
1394 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1396 struct nand_chip *chip = &host->chip;
1397 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1398 struct nand_ecc_ctrl *ecc = &chip->ecc;
1402 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1403 nandc_read_buffer_sync(nandc, true);
1405 for (i = 0; i < num_cw; i++) {
1406 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1408 if (flash_status & FS_MPU_ERR)
1409 host->status &= ~NAND_STATUS_WP;
1411 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1413 FS_DEVICE_STS_ERR)))
1414 host->status |= NAND_STATUS_FAIL;
1418 static void post_command(struct qcom_nand_host *host, int command)
1420 struct nand_chip *chip = &host->chip;
1421 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1424 case NAND_CMD_READID:
1425 nandc_read_buffer_sync(nandc, true);
1426 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1429 case NAND_CMD_PAGEPROG:
1430 case NAND_CMD_ERASE1:
1431 parse_erase_write_errors(host, command);
1439 * Implements chip->cmdfunc. It's only used for a limited set of commands.
1440 * The rest of the commands wouldn't be called by upper layers. For example,
1441 * NAND_CMD_READOOB would never be called because we have our own versions
1442 * of read_oob ops for nand_ecc_ctrl.
1444 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
1445 int column, int page_addr)
1447 struct nand_chip *chip = mtd_to_nand(mtd);
1448 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1449 struct nand_ecc_ctrl *ecc = &chip->ecc;
1450 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1454 pre_command(host, command);
1457 case NAND_CMD_RESET:
1462 case NAND_CMD_READID:
1463 nandc->buf_count = 4;
1464 ret = read_id(host, column);
1468 case NAND_CMD_PARAM:
1469 ret = nandc_param(host);
1473 case NAND_CMD_ERASE1:
1474 ret = erase_block(host, page_addr);
1478 case NAND_CMD_READ0:
1479 /* we read the entire page for now */
1480 WARN_ON(column != 0);
1482 host->use_ecc = true;
1483 set_address(host, 0, page_addr);
1484 update_rw_regs(host, ecc->steps, true);
1487 case NAND_CMD_SEQIN:
1488 WARN_ON(column != 0);
1489 set_address(host, 0, page_addr);
1492 case NAND_CMD_PAGEPROG:
1493 case NAND_CMD_STATUS:
1500 dev_err(nandc->dev, "failure executing command %d\n",
1507 ret = submit_descs(nandc);
1510 "failure submitting descs for command %d\n",
1516 post_command(host, command);
1520 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1521 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1523 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1524 * but it notifies that it is an erased CW by placing special characters at
1525 * certain offsets in the buffer.
1527 * verify if the page is erased or not, and fix up the page for RS ECC by
1528 * replacing the special characters with 0xff.
1530 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1535 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1536 * is erased by looking for 0x54s at offsets 3 and 175 from the
1537 * beginning of each codeword
1540 empty1 = data_buf[3];
1541 empty2 = data_buf[175];
1544 * if the erased codework markers, if they exist override them with
1547 if ((empty1 == 0x54 && empty2 == 0xff) ||
1548 (empty1 == 0xff && empty2 == 0x54)) {
1550 data_buf[175] = 0xff;
1554 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1555 * restore the original values at the special offsets
1557 if (memchr_inv(data_buf, 0xff, data_len)) {
1558 data_buf[3] = empty1;
1559 data_buf[175] = empty2;
1573 /* reads back FLASH_STATUS register set by the controller */
1574 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1576 struct nand_chip *chip = &host->chip;
1577 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1580 nandc_read_buffer_sync(nandc, true);
1582 for (i = 0; i < cw_cnt; i++) {
1583 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1585 if (flash & (FS_OP_ERR | FS_MPU_ERR))
1592 /* performs raw read for one codeword */
1594 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1595 u8 *data_buf, u8 *oob_buf, int page, int cw)
1597 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1598 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1599 struct nand_ecc_ctrl *ecc = &chip->ecc;
1600 int data_size1, data_size2, oob_size1, oob_size2;
1601 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1603 nand_read_page_op(chip, page, 0, NULL, 0);
1604 host->use_ecc = false;
1606 clear_bam_transaction(nandc);
1607 set_address(host, host->cw_size * cw, page);
1608 update_rw_regs(host, 1, true);
1609 config_nand_page_read(nandc);
1611 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1612 oob_size1 = host->bbm_size;
1614 if (cw == (ecc->steps - 1)) {
1615 data_size2 = ecc->size - data_size1 -
1616 ((ecc->steps - 1) * 4);
1617 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1620 data_size2 = host->cw_data - data_size1;
1621 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1624 if (nandc->props->is_bam) {
1625 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1626 read_loc += data_size1;
1628 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1629 read_loc += oob_size1;
1631 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1632 read_loc += data_size2;
1634 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1637 config_nand_cw_read(nandc, false);
1639 read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1640 reg_off += data_size1;
1642 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1643 reg_off += oob_size1;
1645 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1646 reg_off += data_size2;
1648 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1650 ret = submit_descs(nandc);
1653 dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1657 return check_flash_errors(host, 1);
1661 * Bitflips can happen in erased codewords also so this function counts the
1662 * number of 0 in each CW for which ECC engine returns the uncorrectable
1663 * error. The page will be assumed as erased if this count is less than or
1664 * equal to the ecc->strength for each CW.
1666 * 1. Both DATA and OOB need to be checked for number of 0. The
1667 * top-level API can be called with only data buf or OOB buf so use
1668 * chip->data_buf if data buf is null and chip->oob_poi if oob buf
1669 * is null for copying the raw bytes.
1670 * 2. Perform raw read for all the CW which has uncorrectable errors.
1671 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1672 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1673 * the number of bitflips in this area.
1676 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1677 u8 *oob_buf, unsigned long uncorrectable_cws,
1678 int page, unsigned int max_bitflips)
1680 struct nand_chip *chip = &host->chip;
1681 struct mtd_info *mtd = nand_to_mtd(chip);
1682 struct nand_ecc_ctrl *ecc = &chip->ecc;
1683 u8 *cw_data_buf, *cw_oob_buf;
1684 int cw, data_size, oob_size, ret = 0;
1687 data_buf = chip->data_buf;
1692 oob_buf = chip->oob_poi;
1696 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1697 if (cw == (ecc->steps - 1)) {
1698 data_size = ecc->size - ((ecc->steps - 1) * 4);
1699 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1701 data_size = host->cw_data;
1702 oob_size = host->ecc_bytes_hw;
1705 /* determine starting buffer address for current CW */
1706 cw_data_buf = data_buf + (cw * host->cw_data);
1707 cw_oob_buf = oob_buf + (cw * ecc->bytes);
1709 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1710 cw_oob_buf, page, cw);
1715 * make sure it isn't an erased page reported
1716 * as not-erased by HW because of a few bitflips
1718 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1719 cw_oob_buf + host->bbm_size,
1723 mtd->ecc_stats.failed++;
1725 mtd->ecc_stats.corrected += ret;
1726 max_bitflips = max_t(unsigned int, max_bitflips, ret);
1730 return max_bitflips;
1734 * reads back status registers set by the controller to notify page read
1735 * errors. this is equivalent to what 'ecc->correct()' would do.
1737 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1738 u8 *oob_buf, int page)
1740 struct nand_chip *chip = &host->chip;
1741 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1742 struct mtd_info *mtd = nand_to_mtd(chip);
1743 struct nand_ecc_ctrl *ecc = &chip->ecc;
1744 unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1745 struct read_stats *buf;
1746 bool flash_op_err = false, erased;
1748 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1750 buf = (struct read_stats *)nandc->reg_read_buf;
1751 nandc_read_buffer_sync(nandc, true);
1753 for (i = 0; i < ecc->steps; i++, buf++) {
1754 u32 flash, buffer, erased_cw;
1755 int data_len, oob_len;
1757 if (i == (ecc->steps - 1)) {
1758 data_len = ecc->size - ((ecc->steps - 1) << 2);
1759 oob_len = ecc->steps << 2;
1761 data_len = host->cw_data;
1765 flash = le32_to_cpu(buf->flash);
1766 buffer = le32_to_cpu(buf->buffer);
1767 erased_cw = le32_to_cpu(buf->erased_cw);
1770 * Check ECC failure for each codeword. ECC failure can
1771 * happen in either of the following conditions
1772 * 1. If number of bitflips are greater than ECC engine
1774 * 2. If this codeword contains all 0xff for which erased
1775 * codeword detection check will be done.
1777 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1779 * For BCH ECC, ignore erased codeword errors, if
1780 * ERASED_CW bits are set.
1782 if (host->bch_enabled) {
1783 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1786 * For RS ECC, HW reports the erased CW by placing
1787 * special characters at certain offsets in the buffer.
1788 * These special characters will be valid only if
1789 * complete page is read i.e. data_buf is not NULL.
1791 } else if (data_buf) {
1792 erased = erased_chunk_check_and_fixup(data_buf,
1799 uncorrectable_cws |= BIT(i);
1801 * Check if MPU or any other operational error (timeout,
1802 * device failure, etc.) happened for this codeword and
1803 * make flash_op_err true. If flash_op_err is set, then
1804 * EIO will be returned for page read.
1806 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1807 flash_op_err = true;
1809 * No ECC or operational errors happened. Check the number of
1810 * bits corrected and update the ecc_stats.corrected.
1815 stat = buffer & BS_CORRECTABLE_ERR_MSK;
1816 mtd->ecc_stats.corrected += stat;
1817 max_bitflips = max(max_bitflips, stat);
1821 data_buf += data_len;
1823 oob_buf += oob_len + ecc->bytes;
1829 if (!uncorrectable_cws)
1830 return max_bitflips;
1832 return check_for_erased_page(host, data_buf_start, oob_buf_start,
1833 uncorrectable_cws, page,
1838 * helper to perform the actual page read operation, used by ecc->read_page(),
1841 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1842 u8 *oob_buf, int page)
1844 struct nand_chip *chip = &host->chip;
1845 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1846 struct nand_ecc_ctrl *ecc = &chip->ecc;
1847 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1850 config_nand_page_read(nandc);
1852 /* queue cmd descs for each codeword */
1853 for (i = 0; i < ecc->steps; i++) {
1854 int data_size, oob_size;
1856 if (i == (ecc->steps - 1)) {
1857 data_size = ecc->size - ((ecc->steps - 1) << 2);
1858 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1861 data_size = host->cw_data;
1862 oob_size = host->ecc_bytes_hw + host->spare_bytes;
1865 if (nandc->props->is_bam) {
1866 if (data_buf && oob_buf) {
1867 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1868 nandc_set_read_loc(nandc, 1, data_size,
1870 } else if (data_buf) {
1871 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1873 nandc_set_read_loc(nandc, 0, data_size,
1878 config_nand_cw_read(nandc, true);
1881 read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1885 * when ecc is enabled, the controller doesn't read the real
1886 * or dummy bad block markers in each chunk. To maintain a
1887 * consistent layout across RAW and ECC reads, we just
1888 * leave the real/dummy BBM offsets empty (i.e, filled with
1894 for (j = 0; j < host->bbm_size; j++)
1897 read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1898 oob_buf, oob_size, 0);
1902 data_buf += data_size;
1904 oob_buf += oob_size;
1907 ret = submit_descs(nandc);
1911 dev_err(nandc->dev, "failure to read page/oob\n");
1915 return parse_read_errors(host, data_buf_start, oob_buf_start, page);
1919 * a helper that copies the last step/codeword of a page (containing free oob)
1920 * into our local buffer
1922 static int copy_last_cw(struct qcom_nand_host *host, int page)
1924 struct nand_chip *chip = &host->chip;
1925 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1926 struct nand_ecc_ctrl *ecc = &chip->ecc;
1930 clear_read_regs(nandc);
1932 size = host->use_ecc ? host->cw_data : host->cw_size;
1934 /* prepare a clean read buffer */
1935 memset(nandc->data_buffer, 0xff, size);
1937 set_address(host, host->cw_size * (ecc->steps - 1), page);
1938 update_rw_regs(host, 1, true);
1940 config_nand_single_cw_page_read(nandc, host->use_ecc);
1942 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1944 ret = submit_descs(nandc);
1946 dev_err(nandc->dev, "failed to copy last codeword\n");
1953 /* implements ecc->read_page() */
1954 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1955 uint8_t *buf, int oob_required, int page)
1957 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1958 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1959 u8 *data_buf, *oob_buf = NULL;
1961 nand_read_page_op(chip, page, 0, NULL, 0);
1963 oob_buf = oob_required ? chip->oob_poi : NULL;
1965 clear_bam_transaction(nandc);
1967 return read_page_ecc(host, data_buf, oob_buf, page);
1970 /* implements ecc->read_page_raw() */
1971 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1972 struct nand_chip *chip, uint8_t *buf,
1973 int oob_required, int page)
1975 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1976 struct nand_ecc_ctrl *ecc = &chip->ecc;
1978 u8 *data_buf = buf, *oob_buf = chip->oob_poi;
1980 for (cw = 0; cw < ecc->steps; cw++) {
1981 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
1986 data_buf += host->cw_data;
1987 oob_buf += ecc->bytes;
1993 /* implements ecc->read_oob() */
1994 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1997 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1998 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1999 struct nand_ecc_ctrl *ecc = &chip->ecc;
2001 clear_read_regs(nandc);
2002 clear_bam_transaction(nandc);
2004 host->use_ecc = true;
2005 set_address(host, 0, page);
2006 update_rw_regs(host, ecc->steps, true);
2008 return read_page_ecc(host, NULL, chip->oob_poi, page);
2011 /* implements ecc->write_page() */
2012 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
2013 const uint8_t *buf, int oob_required, int page)
2015 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2016 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2017 struct nand_ecc_ctrl *ecc = &chip->ecc;
2018 u8 *data_buf, *oob_buf;
2021 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2023 clear_read_regs(nandc);
2024 clear_bam_transaction(nandc);
2026 data_buf = (u8 *)buf;
2027 oob_buf = chip->oob_poi;
2029 host->use_ecc = true;
2030 update_rw_regs(host, ecc->steps, false);
2031 config_nand_page_write(nandc);
2033 for (i = 0; i < ecc->steps; i++) {
2034 int data_size, oob_size;
2036 if (i == (ecc->steps - 1)) {
2037 data_size = ecc->size - ((ecc->steps - 1) << 2);
2038 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2041 data_size = host->cw_data;
2042 oob_size = ecc->bytes;
2046 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2047 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2050 * when ECC is enabled, we don't really need to write anything
2051 * to oob for the first n - 1 codewords since these oob regions
2052 * just contain ECC bytes that's written by the controller
2053 * itself. For the last codeword, we skip the bbm positions and
2054 * write to the free oob area.
2056 if (i == (ecc->steps - 1)) {
2057 oob_buf += host->bbm_size;
2059 write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2060 oob_buf, oob_size, 0);
2063 config_nand_cw_write(nandc);
2065 data_buf += data_size;
2066 oob_buf += oob_size;
2069 ret = submit_descs(nandc);
2071 dev_err(nandc->dev, "failure to write page\n");
2076 ret = nand_prog_page_end_op(chip);
2081 /* implements ecc->write_page_raw() */
2082 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
2083 struct nand_chip *chip, const uint8_t *buf,
2084 int oob_required, int page)
2086 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2087 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2088 struct nand_ecc_ctrl *ecc = &chip->ecc;
2089 u8 *data_buf, *oob_buf;
2092 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2093 clear_read_regs(nandc);
2094 clear_bam_transaction(nandc);
2096 data_buf = (u8 *)buf;
2097 oob_buf = chip->oob_poi;
2099 host->use_ecc = false;
2100 update_rw_regs(host, ecc->steps, false);
2101 config_nand_page_write(nandc);
2103 for (i = 0; i < ecc->steps; i++) {
2104 int data_size1, data_size2, oob_size1, oob_size2;
2105 int reg_off = FLASH_BUF_ACC;
2107 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2108 oob_size1 = host->bbm_size;
2110 if (i == (ecc->steps - 1)) {
2111 data_size2 = ecc->size - data_size1 -
2112 ((ecc->steps - 1) << 2);
2113 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2116 data_size2 = host->cw_data - data_size1;
2117 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2120 write_data_dma(nandc, reg_off, data_buf, data_size1,
2122 reg_off += data_size1;
2123 data_buf += data_size1;
2125 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2127 reg_off += oob_size1;
2128 oob_buf += oob_size1;
2130 write_data_dma(nandc, reg_off, data_buf, data_size2,
2132 reg_off += data_size2;
2133 data_buf += data_size2;
2135 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2136 oob_buf += oob_size2;
2138 config_nand_cw_write(nandc);
2141 ret = submit_descs(nandc);
2143 dev_err(nandc->dev, "failure to write raw page\n");
2148 ret = nand_prog_page_end_op(chip);
2154 * implements ecc->write_oob()
2156 * the NAND controller cannot write only data or only OOB within a codeword
2157 * since ECC is calculated for the combined codeword. So update the OOB from
2158 * chip->oob_poi, and pad the data area with OxFF before writing.
2160 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
2163 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2164 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2165 struct nand_ecc_ctrl *ecc = &chip->ecc;
2166 u8 *oob = chip->oob_poi;
2167 int data_size, oob_size;
2170 host->use_ecc = true;
2171 clear_bam_transaction(nandc);
2173 /* calculate the data and oob size for the last codeword/step */
2174 data_size = ecc->size - ((ecc->steps - 1) << 2);
2175 oob_size = mtd->oobavail;
2177 memset(nandc->data_buffer, 0xff, host->cw_data);
2178 /* override new oob content to last codeword */
2179 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2182 set_address(host, host->cw_size * (ecc->steps - 1), page);
2183 update_rw_regs(host, 1, false);
2185 config_nand_page_write(nandc);
2186 write_data_dma(nandc, FLASH_BUF_ACC,
2187 nandc->data_buffer, data_size + oob_size, 0);
2188 config_nand_cw_write(nandc);
2190 ret = submit_descs(nandc);
2195 dev_err(nandc->dev, "failure to write oob\n");
2199 return nand_prog_page_end_op(chip);
2202 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
2204 struct nand_chip *chip = mtd_to_nand(mtd);
2205 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2206 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2207 struct nand_ecc_ctrl *ecc = &chip->ecc;
2208 int page, ret, bbpos, bad = 0;
2210 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2213 * configure registers for a raw sub page read, the address is set to
2214 * the beginning of the last codeword, we don't care about reading ecc
2215 * portion of oob. we just want the first few bytes from this codeword
2216 * that contains the BBM
2218 host->use_ecc = false;
2220 clear_bam_transaction(nandc);
2221 ret = copy_last_cw(host, page);
2225 if (check_flash_errors(host, 1)) {
2226 dev_warn(nandc->dev, "error when trying to read BBM\n");
2230 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2232 bad = nandc->data_buffer[bbpos] != 0xff;
2234 if (chip->options & NAND_BUSWIDTH_16)
2235 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2240 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
2242 struct nand_chip *chip = mtd_to_nand(mtd);
2243 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2244 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2245 struct nand_ecc_ctrl *ecc = &chip->ecc;
2248 clear_read_regs(nandc);
2249 clear_bam_transaction(nandc);
2252 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2253 * we don't care about the rest of the content in the codeword since
2254 * we aren't going to use this block again
2256 memset(nandc->data_buffer, 0x00, host->cw_size);
2258 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2261 host->use_ecc = false;
2262 set_address(host, host->cw_size * (ecc->steps - 1), page);
2263 update_rw_regs(host, 1, false);
2265 config_nand_page_write(nandc);
2266 write_data_dma(nandc, FLASH_BUF_ACC,
2267 nandc->data_buffer, host->cw_size, 0);
2268 config_nand_cw_write(nandc);
2270 ret = submit_descs(nandc);
2275 dev_err(nandc->dev, "failure to update BBM\n");
2279 return nand_prog_page_end_op(chip);
2283 * the three functions below implement chip->read_byte(), chip->read_buf()
2284 * and chip->write_buf() respectively. these aren't used for
2285 * reading/writing page data, they are used for smaller data like reading
2288 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
2290 struct nand_chip *chip = mtd_to_nand(mtd);
2291 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2292 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2293 u8 *buf = nandc->data_buffer;
2296 if (host->last_command == NAND_CMD_STATUS) {
2299 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2304 if (nandc->buf_start < nandc->buf_count)
2305 ret = buf[nandc->buf_start++];
2310 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
2312 struct nand_chip *chip = mtd_to_nand(mtd);
2313 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2314 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2316 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2317 nandc->buf_start += real_len;
2320 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
2323 struct nand_chip *chip = mtd_to_nand(mtd);
2324 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2325 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2327 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2329 nandc->buf_start += real_len;
2332 /* we support only one external chip for now */
2333 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
2335 struct nand_chip *chip = mtd_to_nand(mtd);
2336 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2341 dev_warn(nandc->dev, "invalid chip select\n");
2345 * NAND controller page layout info
2347 * Layout with ECC enabled:
2349 * |----------------------| |---------------------------------|
2350 * | xx.......yy| | *********xx.......yy|
2351 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
2352 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
2353 * | xx.......yy| | *********xx.......yy|
2354 * |----------------------| |---------------------------------|
2355 * codeword 1,2..n-1 codeword n
2356 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
2358 * n = Number of codewords in the page
2360 * * = Spare/free bytes
2361 * x = Unused byte(s)
2362 * y = Reserved byte(s)
2364 * 2K page: n = 4, spare = 16 bytes
2365 * 4K page: n = 8, spare = 32 bytes
2366 * 8K page: n = 16, spare = 64 bytes
2368 * the qcom nand controller operates at a sub page/codeword level. each
2369 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2370 * the number of ECC bytes vary based on the ECC strength and the bus width.
2372 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2373 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2374 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2376 * When we access a page with ECC enabled, the reserved bytes(s) are not
2377 * accessible at all. When reading, we fill up these unreadable positions
2378 * with 0xffs. When writing, the controller skips writing the inaccessible
2381 * Layout with ECC disabled:
2383 * |------------------------------| |---------------------------------------|
2384 * | yy xx.......| | bb *********xx.......|
2385 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
2386 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
2387 * | yy xx.......| | bb *********xx.......|
2388 * |------------------------------| |---------------------------------------|
2389 * codeword 1,2..n-1 codeword n
2390 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
2392 * n = Number of codewords in the page
2394 * * = Spare/free bytes
2395 * x = Unused byte(s)
2396 * y = Dummy Bad Bock byte(s)
2397 * b = Real Bad Block byte(s)
2398 * size1/size2 = function of codeword size and 'n'
2400 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2401 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2402 * Block Markers. In the last codeword, this position contains the real BBM
2404 * In order to have a consistent layout between RAW and ECC modes, we assume
2405 * the following OOB layout arrangement:
2407 * |-----------| |--------------------|
2408 * |yyxx.......| |bb*********xx.......|
2409 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
2410 * |yyxx.......| |bb*********xx.......|
2411 * |yyxx.......| |bb*********xx.......|
2412 * |-----------| |--------------------|
2413 * first n - 1 nth OOB region
2416 * n = Number of codewords in the page
2418 * * = FREE OOB bytes
2419 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2420 * x = Unused byte(s)
2421 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2423 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2424 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2425 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2426 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2427 * the sum of the three).
2429 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2430 struct mtd_oob_region *oobregion)
2432 struct nand_chip *chip = mtd_to_nand(mtd);
2433 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2434 struct nand_ecc_ctrl *ecc = &chip->ecc;
2440 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2442 oobregion->offset = 0;
2444 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2445 oobregion->offset = mtd->oobsize - oobregion->length;
2451 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2452 struct mtd_oob_region *oobregion)
2454 struct nand_chip *chip = mtd_to_nand(mtd);
2455 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2456 struct nand_ecc_ctrl *ecc = &chip->ecc;
2461 oobregion->length = ecc->steps * 4;
2462 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2467 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2468 .ecc = qcom_nand_ooblayout_ecc,
2469 .free = qcom_nand_ooblayout_free,
2473 qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2475 return strength == 4 ? 12 : 16;
2477 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2478 NANDC_STEP_SIZE, 4, 8);
2480 static int qcom_nand_attach_chip(struct nand_chip *chip)
2482 struct mtd_info *mtd = nand_to_mtd(chip);
2483 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2484 struct nand_ecc_ctrl *ecc = &chip->ecc;
2485 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2486 int cwperpage, bad_block_byte, ret;
2490 /* controller only supports 512 bytes data steps */
2491 ecc->size = NANDC_STEP_SIZE;
2492 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2493 cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2496 * Each CW has 4 available OOB bytes which will be protected with ECC
2497 * so remaining bytes can be used for ECC.
2499 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2500 mtd->oobsize - (cwperpage * 4));
2502 dev_err(nandc->dev, "No valid ECC settings possible\n");
2506 if (ecc->strength >= 8) {
2507 /* 8 bit ECC defaults to BCH ECC on all platforms */
2508 host->bch_enabled = true;
2512 host->ecc_bytes_hw = 14;
2513 host->spare_bytes = 0;
2516 host->ecc_bytes_hw = 13;
2517 host->spare_bytes = 2;
2522 * if the controller supports BCH for 4 bit ECC, the controller
2523 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2526 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2528 host->bch_enabled = true;
2532 host->ecc_bytes_hw = 8;
2533 host->spare_bytes = 2;
2536 host->ecc_bytes_hw = 7;
2537 host->spare_bytes = 4;
2542 host->ecc_bytes_hw = 10;
2545 host->spare_bytes = 0;
2548 host->spare_bytes = 1;
2555 * we consider ecc->bytes as the sum of all the non-data content in a
2556 * step. It gives us a clean representation of the oob area (even if
2557 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2558 * ECC and 12 bytes for 4 bit ECC
2560 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2562 ecc->read_page = qcom_nandc_read_page;
2563 ecc->read_page_raw = qcom_nandc_read_page_raw;
2564 ecc->read_oob = qcom_nandc_read_oob;
2565 ecc->write_page = qcom_nandc_write_page;
2566 ecc->write_page_raw = qcom_nandc_write_page_raw;
2567 ecc->write_oob = qcom_nandc_write_oob;
2569 ecc->mode = NAND_ECC_HW;
2571 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2573 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2577 * DATA_UD_BYTES varies based on whether the read/write command protects
2578 * spare data with ECC too. We protect spare data by default, so we set
2579 * it to main + spare data, which are 512 and 4 bytes respectively.
2581 host->cw_data = 516;
2584 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2587 host->cw_size = host->cw_data + ecc->bytes;
2588 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2590 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2591 | host->cw_data << UD_SIZE_BYTES
2592 | 0 << DISABLE_STATUS_AFTER_WRITE
2593 | 5 << NUM_ADDR_CYCLES
2594 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2595 | 0 << STATUS_BFR_READ
2596 | 1 << SET_RD_MODE_AFTER_STATUS
2597 | host->spare_bytes << SPARE_SIZE_BYTES;
2599 host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2600 | 0 << CS_ACTIVE_BSY
2601 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2602 | 0 << BAD_BLOCK_IN_SPARE_AREA
2603 | 2 << WR_RD_BSY_GAP
2604 | wide_bus << WIDE_FLASH
2605 | host->bch_enabled << ENABLE_BCH_ECC;
2607 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2608 | host->cw_size << UD_SIZE_BYTES
2609 | 5 << NUM_ADDR_CYCLES
2610 | 0 << SPARE_SIZE_BYTES;
2612 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2613 | 0 << CS_ACTIVE_BSY
2614 | 17 << BAD_BLOCK_BYTE_NUM
2615 | 1 << BAD_BLOCK_IN_SPARE_AREA
2616 | 2 << WR_RD_BSY_GAP
2617 | wide_bus << WIDE_FLASH
2618 | 1 << DEV0_CFG1_ECC_DISABLE;
2620 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2622 | host->cw_data << ECC_NUM_DATA_BYTES
2623 | 1 << ECC_FORCE_CLK_OPEN
2624 | ecc_mode << ECC_MODE
2625 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2627 host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2629 host->clrflashstatus = FS_READY_BSY_N;
2630 host->clrreadstatus = 0xc0;
2631 nandc->regs->erased_cw_detect_cfg_clr =
2632 cpu_to_le32(CLR_ERASED_PAGE_DET);
2633 nandc->regs->erased_cw_detect_cfg_set =
2634 cpu_to_le32(SET_ERASED_PAGE_DET);
2637 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2638 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2639 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2645 static const struct nand_controller_ops qcom_nandc_ops = {
2646 .attach_chip = qcom_nand_attach_chip,
2649 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2653 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2655 dev_err(nandc->dev, "failed to set DMA mask\n");
2660 * we use the internal buffer for reading ONFI params, reading small
2661 * data like ID and status, and preforming read-copy-write operations
2662 * when writing to a codeword partially. 532 is the maximum possible
2663 * size of a codeword for our nand controller
2665 nandc->buf_size = 532;
2667 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2669 if (!nandc->data_buffer)
2672 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2677 nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2678 MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2680 if (!nandc->reg_read_buf)
2683 if (nandc->props->is_bam) {
2684 nandc->reg_read_dma =
2685 dma_map_single(nandc->dev, nandc->reg_read_buf,
2687 sizeof(*nandc->reg_read_buf),
2689 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2690 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2694 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2695 if (!nandc->tx_chan) {
2696 dev_err(nandc->dev, "failed to request tx channel\n");
2700 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2701 if (!nandc->rx_chan) {
2702 dev_err(nandc->dev, "failed to request rx channel\n");
2706 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2707 if (!nandc->cmd_chan) {
2708 dev_err(nandc->dev, "failed to request cmd channel\n");
2713 * Initially allocate BAM transaction to read ONFI param page.
2714 * After detecting all the devices, this BAM transaction will
2715 * be freed and the next BAM tranasction will be allocated with
2716 * maximum codeword size
2718 nandc->max_cwperpage = 1;
2719 nandc->bam_txn = alloc_bam_transaction(nandc);
2720 if (!nandc->bam_txn) {
2722 "failed to allocate bam transaction\n");
2726 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2729 "failed to request slave channel\n");
2734 INIT_LIST_HEAD(&nandc->desc_list);
2735 INIT_LIST_HEAD(&nandc->host_list);
2737 nand_controller_init(&nandc->controller);
2738 nandc->controller.ops = &qcom_nandc_ops;
2743 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2745 if (nandc->props->is_bam) {
2746 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2747 dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2749 sizeof(*nandc->reg_read_buf),
2753 dma_release_channel(nandc->tx_chan);
2756 dma_release_channel(nandc->rx_chan);
2758 if (nandc->cmd_chan)
2759 dma_release_channel(nandc->cmd_chan);
2762 dma_release_channel(nandc->chan);
2766 /* one time setup of a few nand controller registers */
2767 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2772 if (!nandc->props->is_qpic)
2773 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2774 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2775 NAND_DEV_CMD_VLD_VAL);
2777 /* enable ADM or BAM DMA */
2778 if (nandc->props->is_bam) {
2779 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2780 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2782 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2785 /* save the original values of these registers */
2786 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2787 nandc->vld = NAND_DEV_CMD_VLD_VAL;
2792 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2793 struct qcom_nand_host *host,
2794 struct device_node *dn)
2796 struct nand_chip *chip = &host->chip;
2797 struct mtd_info *mtd = nand_to_mtd(chip);
2798 struct device *dev = nandc->dev;
2801 ret = of_property_read_u32(dn, "reg", &host->cs);
2803 dev_err(dev, "can't get chip-select\n");
2807 nand_set_flash_node(chip, dn);
2808 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2812 mtd->owner = THIS_MODULE;
2813 mtd->dev.parent = dev;
2815 chip->cmdfunc = qcom_nandc_command;
2816 chip->select_chip = qcom_nandc_select_chip;
2817 chip->read_byte = qcom_nandc_read_byte;
2818 chip->read_buf = qcom_nandc_read_buf;
2819 chip->write_buf = qcom_nandc_write_buf;
2820 chip->set_features = nand_get_set_features_notsupp;
2821 chip->get_features = nand_get_set_features_notsupp;
2824 * the bad block marker is readable only when we read the last codeword
2825 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2826 * helpers don't allow us to read BB from a nand chip with ECC
2827 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2828 * and block_markbad helpers until we permanently switch to using
2829 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2831 chip->block_bad = qcom_nandc_block_bad;
2832 chip->block_markbad = qcom_nandc_block_markbad;
2834 chip->controller = &nandc->controller;
2835 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2838 /* set up initial status value */
2839 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2841 ret = nand_scan(chip, 1);
2845 if (nandc->props->is_bam) {
2846 free_bam_transaction(nandc);
2847 nandc->bam_txn = alloc_bam_transaction(nandc);
2848 if (!nandc->bam_txn) {
2850 "failed to allocate bam transaction\n");
2855 ret = mtd_device_register(mtd, NULL, 0);
2862 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2864 struct device *dev = nandc->dev;
2865 struct device_node *dn = dev->of_node, *child;
2866 struct qcom_nand_host *host;
2869 for_each_available_child_of_node(dn, child) {
2870 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2876 ret = qcom_nand_host_init_and_register(nandc, host, child);
2878 devm_kfree(dev, host);
2882 list_add_tail(&host->node, &nandc->host_list);
2888 /* parse custom DT properties here */
2889 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2891 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2892 struct device_node *np = nandc->dev->of_node;
2895 if (!nandc->props->is_bam) {
2896 ret = of_property_read_u32(np, "qcom,cmd-crci",
2899 dev_err(nandc->dev, "command CRCI unspecified\n");
2903 ret = of_property_read_u32(np, "qcom,data-crci",
2906 dev_err(nandc->dev, "data CRCI unspecified\n");
2914 static int qcom_nandc_probe(struct platform_device *pdev)
2916 struct qcom_nand_controller *nandc;
2917 const void *dev_data;
2918 struct device *dev = &pdev->dev;
2919 struct resource *res;
2922 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2926 platform_set_drvdata(pdev, nandc);
2929 dev_data = of_device_get_match_data(dev);
2931 dev_err(&pdev->dev, "failed to get device data\n");
2935 nandc->props = dev_data;
2937 nandc->core_clk = devm_clk_get(dev, "core");
2938 if (IS_ERR(nandc->core_clk))
2939 return PTR_ERR(nandc->core_clk);
2941 nandc->aon_clk = devm_clk_get(dev, "aon");
2942 if (IS_ERR(nandc->aon_clk))
2943 return PTR_ERR(nandc->aon_clk);
2945 ret = qcom_nandc_parse_dt(pdev);
2949 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2950 nandc->base = devm_ioremap_resource(dev, res);
2951 if (IS_ERR(nandc->base))
2952 return PTR_ERR(nandc->base);
2954 nandc->base_phys = res->start;
2955 nandc->base_dma = dma_map_resource(dev, res->start,
2957 DMA_BIDIRECTIONAL, 0);
2958 if (!nandc->base_dma)
2961 ret = clk_prepare_enable(nandc->core_clk);
2965 ret = clk_prepare_enable(nandc->aon_clk);
2969 ret = qcom_nandc_alloc(nandc);
2971 goto err_nandc_alloc;
2973 ret = qcom_nandc_setup(nandc);
2977 ret = qcom_probe_nand_devices(nandc);
2984 qcom_nandc_unalloc(nandc);
2986 clk_disable_unprepare(nandc->aon_clk);
2988 clk_disable_unprepare(nandc->core_clk);
2990 dma_unmap_resource(dev, res->start, resource_size(res),
2991 DMA_BIDIRECTIONAL, 0);
2995 static int qcom_nandc_remove(struct platform_device *pdev)
2997 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2998 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2999 struct qcom_nand_host *host;
3001 list_for_each_entry(host, &nandc->host_list, node)
3002 nand_release(&host->chip);
3005 qcom_nandc_unalloc(nandc);
3007 clk_disable_unprepare(nandc->aon_clk);
3008 clk_disable_unprepare(nandc->core_clk);
3010 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3011 DMA_BIDIRECTIONAL, 0);
3016 static const struct qcom_nandc_props ipq806x_nandc_props = {
3017 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3019 .dev_cmd_reg_start = 0x0,
3022 static const struct qcom_nandc_props ipq4019_nandc_props = {
3023 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3026 .dev_cmd_reg_start = 0x0,
3029 static const struct qcom_nandc_props ipq8074_nandc_props = {
3030 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3033 .dev_cmd_reg_start = 0x7000,
3037 * data will hold a struct pointer containing more differences once we support
3038 * more controller variants
3040 static const struct of_device_id qcom_nandc_of_match[] = {
3042 .compatible = "qcom,ipq806x-nand",
3043 .data = &ipq806x_nandc_props,
3046 .compatible = "qcom,ipq4019-nand",
3047 .data = &ipq4019_nandc_props,
3050 .compatible = "qcom,ipq8074-nand",
3051 .data = &ipq8074_nandc_props,
3055 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3057 static struct platform_driver qcom_nandc_driver = {
3059 .name = "qcom-nandc",
3060 .of_match_table = qcom_nandc_of_match,
3062 .probe = qcom_nandc_probe,
3063 .remove = qcom_nandc_remove,
3065 module_platform_driver(qcom_nandc_driver);
3067 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3068 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3069 MODULE_LICENSE("GPL v2");