2 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
14 #include <linux/clk.h>
15 #include <linux/slab.h>
16 #include <linux/bitops.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmaengine.h>
19 #include <linux/module.h>
20 #include <linux/mtd/rawnand.h>
21 #include <linux/mtd/partitions.h>
23 #include <linux/of_device.h>
24 #include <linux/delay.h>
25 #include <linux/dma/qcom_bam_dma.h>
27 /* NANDc reg offsets */
28 #define NAND_FLASH_CMD 0x00
29 #define NAND_ADDR0 0x04
30 #define NAND_ADDR1 0x08
31 #define NAND_FLASH_CHIP_SELECT 0x0c
32 #define NAND_EXEC_CMD 0x10
33 #define NAND_FLASH_STATUS 0x14
34 #define NAND_BUFFER_STATUS 0x18
35 #define NAND_DEV0_CFG0 0x20
36 #define NAND_DEV0_CFG1 0x24
37 #define NAND_DEV0_ECC_CFG 0x28
38 #define NAND_DEV1_ECC_CFG 0x2c
39 #define NAND_DEV1_CFG0 0x30
40 #define NAND_DEV1_CFG1 0x34
41 #define NAND_READ_ID 0x40
42 #define NAND_READ_STATUS 0x44
43 #define NAND_DEV_CMD0 0xa0
44 #define NAND_DEV_CMD1 0xa4
45 #define NAND_DEV_CMD2 0xa8
46 #define NAND_DEV_CMD_VLD 0xac
47 #define SFLASHC_BURST_CFG 0xe0
48 #define NAND_ERASED_CW_DETECT_CFG 0xe8
49 #define NAND_ERASED_CW_DETECT_STATUS 0xec
50 #define NAND_EBI2_ECC_BUF_CFG 0xf0
51 #define FLASH_BUF_ACC 0x100
53 #define NAND_CTRL 0xf00
54 #define NAND_VERSION 0xf08
55 #define NAND_READ_LOCATION_0 0xf20
56 #define NAND_READ_LOCATION_1 0xf24
57 #define NAND_READ_LOCATION_2 0xf28
58 #define NAND_READ_LOCATION_3 0xf2c
60 /* dummy register offsets, used by write_reg_dma */
61 #define NAND_DEV_CMD1_RESTORE 0xdead
62 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef
64 /* NAND_FLASH_CMD bits */
65 #define PAGE_ACC BIT(4)
66 #define LAST_PAGE BIT(5)
68 /* NAND_FLASH_CHIP_SELECT bits */
69 #define NAND_DEV_SEL 0
72 /* NAND_FLASH_STATUS bits */
73 #define FS_OP_ERR BIT(4)
74 #define FS_READY_BSY_N BIT(5)
75 #define FS_MPU_ERR BIT(8)
76 #define FS_DEVICE_STS_ERR BIT(16)
77 #define FS_DEVICE_WP BIT(23)
79 /* NAND_BUFFER_STATUS bits */
80 #define BS_UNCORRECTABLE_BIT BIT(8)
81 #define BS_CORRECTABLE_ERR_MSK 0x1f
83 /* NAND_DEVn_CFG0 bits */
84 #define DISABLE_STATUS_AFTER_WRITE 4
86 #define UD_SIZE_BYTES 9
87 #define ECC_PARITY_SIZE_BYTES_RS 19
88 #define SPARE_SIZE_BYTES 23
89 #define NUM_ADDR_CYCLES 27
90 #define STATUS_BFR_READ 30
91 #define SET_RD_MODE_AFTER_STATUS 31
93 /* NAND_DEVn_CFG0 bits */
94 #define DEV0_CFG1_ECC_DISABLE 0
96 #define NAND_RECOVERY_CYCLES 2
97 #define CS_ACTIVE_BSY 5
98 #define BAD_BLOCK_BYTE_NUM 6
99 #define BAD_BLOCK_IN_SPARE_AREA 16
100 #define WR_RD_BSY_GAP 17
101 #define ENABLE_BCH_ECC 27
103 /* NAND_DEV0_ECC_CFG bits */
104 #define ECC_CFG_ECC_DISABLE 0
105 #define ECC_SW_RESET 1
107 #define ECC_PARITY_SIZE_BYTES_BCH 8
108 #define ECC_NUM_DATA_BYTES 16
109 #define ECC_FORCE_CLK_OPEN 30
111 /* NAND_DEV_CMD1 bits */
114 /* NAND_DEV_CMD_VLD bits */
115 #define READ_START_VLD BIT(0)
116 #define READ_STOP_VLD BIT(1)
117 #define WRITE_START_VLD BIT(2)
118 #define ERASE_START_VLD BIT(3)
119 #define SEQ_READ_START_VLD BIT(4)
121 /* NAND_EBI2_ECC_BUF_CFG bits */
124 /* NAND_ERASED_CW_DETECT_CFG bits */
125 #define ERASED_CW_ECC_MASK 1
126 #define AUTO_DETECT_RES 0
127 #define MASK_ECC (1 << ERASED_CW_ECC_MASK)
128 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
129 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
130 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
131 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
133 /* NAND_ERASED_CW_DETECT_STATUS bits */
134 #define PAGE_ALL_ERASED BIT(7)
135 #define CODEWORD_ALL_ERASED BIT(6)
136 #define PAGE_ERASED BIT(5)
137 #define CODEWORD_ERASED BIT(4)
138 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
139 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
141 /* NAND_READ_LOCATION_n bits */
142 #define READ_LOCATION_OFFSET 0
143 #define READ_LOCATION_SIZE 16
144 #define READ_LOCATION_LAST 31
147 #define NAND_VERSION_MAJOR_MASK 0xf0000000
148 #define NAND_VERSION_MAJOR_SHIFT 28
149 #define NAND_VERSION_MINOR_MASK 0x0fff0000
150 #define NAND_VERSION_MINOR_SHIFT 16
153 #define OP_PAGE_READ 0x2
154 #define OP_PAGE_READ_WITH_ECC 0x3
155 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4
156 #define OP_PROGRAM_PAGE 0x6
157 #define OP_PAGE_PROGRAM_WITH_ECC 0x7
158 #define OP_PROGRAM_PAGE_SPARE 0x9
159 #define OP_BLOCK_ERASE 0xa
160 #define OP_FETCH_ID 0xb
161 #define OP_RESET_DEVICE 0xd
163 /* Default Value for NAND_DEV_CMD_VLD */
164 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \
165 ERASE_START_VLD | SEQ_READ_START_VLD)
168 #define BAM_MODE_EN BIT(0)
171 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
172 * the driver calls the chunks 'step' or 'codeword' interchangeably
174 #define NANDC_STEP_SIZE 512
177 * the largest page size we support is 8K, this will have 16 steps/codewords
180 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
182 /* we read at most 3 registers per codeword scan */
183 #define MAX_REG_RD (3 * MAX_NUM_STEPS)
185 /* ECC modes supported by the controller */
186 #define ECC_NONE BIT(0)
187 #define ECC_RS_4BIT BIT(1)
188 #define ECC_BCH_4BIT BIT(2)
189 #define ECC_BCH_8BIT BIT(3)
191 #define nandc_set_read_loc(nandc, reg, offset, size, is_last) \
192 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \
193 ((offset) << READ_LOCATION_OFFSET) | \
194 ((size) << READ_LOCATION_SIZE) | \
195 ((is_last) << READ_LOCATION_LAST))
198 * Returns the actual register address for all NAND_DEV_ registers
199 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
201 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
203 /* Returns the NAND register physical address */
204 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
206 /* Returns the dma address for reg read buffer */
207 #define reg_buf_dma_addr(chip, vaddr) \
208 ((chip)->reg_read_dma + \
209 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
211 #define QPIC_PER_CW_CMD_ELEMENTS 32
212 #define QPIC_PER_CW_CMD_SGL 32
213 #define QPIC_PER_CW_DATA_SGL 8
215 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000)
218 * Flags used in DMA descriptor preparation helper functions
219 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
221 /* Don't set the EOT in current tx BAM sgl */
222 #define NAND_BAM_NO_EOT BIT(0)
223 /* Set the NWD flag in current BAM sgl */
224 #define NAND_BAM_NWD BIT(1)
225 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
226 #define NAND_BAM_NEXT_SGL BIT(2)
228 * Erased codeword status is being used two times in single transfer so this
229 * flag will determine the current value of erased codeword status register
231 #define NAND_ERASED_CW_SET BIT(4)
234 * This data type corresponds to the BAM transaction which will be used for all
236 * @bam_ce - the array of BAM command elements
237 * @cmd_sgl - sgl for NAND BAM command pipe
238 * @data_sgl - sgl for NAND BAM consumer/producer pipe
239 * @bam_ce_pos - the index in bam_ce which is available for next sgl
240 * @bam_ce_start - the index in bam_ce which marks the start position ce
241 * for current sgl. It will be used for size calculation
243 * @cmd_sgl_pos - current index in command sgl.
244 * @cmd_sgl_start - start index in command sgl.
245 * @tx_sgl_pos - current index in data sgl for tx.
246 * @tx_sgl_start - start index in data sgl for tx.
247 * @rx_sgl_pos - current index in data sgl for rx.
248 * @rx_sgl_start - start index in data sgl for rx.
249 * @wait_second_completion - wait for second DMA desc completion before making
250 * the NAND transfer completion.
251 * @txn_done - completion for NAND transfer.
252 * @last_data_desc - last DMA desc in data channel (tx/rx).
253 * @last_cmd_desc - last DMA desc in command channel.
255 struct bam_transaction
{
256 struct bam_cmd_element
*bam_ce
;
257 struct scatterlist
*cmd_sgl
;
258 struct scatterlist
*data_sgl
;
267 bool wait_second_completion
;
268 struct completion txn_done
;
269 struct dma_async_tx_descriptor
*last_data_desc
;
270 struct dma_async_tx_descriptor
*last_cmd_desc
;
274 * This data type corresponds to the nand dma descriptor
275 * @list - list for desc_info
276 * @dir - DMA transfer direction
277 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
279 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
280 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
281 * @dma_desc - low level DMA engine descriptor
284 struct list_head node
;
286 enum dma_data_direction dir
;
288 struct scatterlist adm_sgl
;
290 struct scatterlist
*bam_sgl
;
294 struct dma_async_tx_descriptor
*dma_desc
;
298 * holds the current register values that we want to write. acts as a contiguous
299 * chunk of memory which we use to write the controller registers through DMA.
312 __le32 clrflashstatus
;
313 __le32 clrreadstatus
;
322 __le32 read_location0
;
323 __le32 read_location1
;
324 __le32 read_location2
;
325 __le32 read_location3
;
327 __le32 erased_cw_detect_cfg_clr
;
328 __le32 erased_cw_detect_cfg_set
;
332 * NAND controller data struct
334 * @controller: base controller structure
335 * @host_list: list containing all the chips attached to the
337 * @dev: parent device
339 * @base_phys: physical base address of controller registers
340 * @base_dma: dma base address of controller registers
341 * @core_clk: controller clock
342 * @aon_clk: another controller clock
345 * @cmd_crci: ADM DMA CRCI for command flow control
346 * @data_crci: ADM DMA CRCI for data flow control
347 * @desc_list: DMA descriptor list (list of desc_infos)
349 * @data_buffer: our local DMA buffer for page read/writes,
350 * used when we can't use the buffer provided
351 * by upper layers directly
352 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf
354 * @reg_read_buf: local buffer for reading back registers via DMA
355 * @reg_read_dma: contains dma address for register read buffer
356 * @reg_read_pos: marker for data read in reg_read_buf
358 * @regs: a contiguous chunk of memory for DMA register
359 * writes. contains the register values to be
360 * written to controller
361 * @cmd1/vld: some fixed controller register values
362 * @props: properties of current NAND controller,
363 * initialized via DT match data
364 * @max_cwperpage: maximum QPIC codewords required. calculated
365 * from all connected NAND devices pagesize
367 struct qcom_nand_controller
{
368 struct nand_controller controller
;
369 struct list_head host_list
;
374 phys_addr_t base_phys
;
377 struct clk
*core_clk
;
381 /* will be used only by QPIC for BAM DMA */
383 struct dma_chan
*tx_chan
;
384 struct dma_chan
*rx_chan
;
385 struct dma_chan
*cmd_chan
;
388 /* will be used only by EBI2 for ADM DMA */
390 struct dma_chan
*chan
;
391 unsigned int cmd_crci
;
392 unsigned int data_crci
;
396 struct list_head desc_list
;
397 struct bam_transaction
*bam_txn
;
403 unsigned int max_cwperpage
;
405 __le32
*reg_read_buf
;
406 dma_addr_t reg_read_dma
;
409 struct nandc_regs
*regs
;
412 const struct qcom_nandc_props
*props
;
416 * NAND chip structure
418 * @chip: base NAND chip structure
419 * @node: list node to add itself to host_list in
420 * qcom_nand_controller
422 * @cs: chip select value for this chip
423 * @cw_size: the number of bytes in a single step/codeword
424 * of a page, consisting of all data, ecc, spare
426 * @cw_data: the number of bytes within a codeword protected
428 * @use_ecc: request the controller to use ECC for the
429 * upcoming read/write
430 * @bch_enabled: flag to tell whether BCH ECC mode is used
431 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
433 * @status: value to be returned if NAND_CMD_STATUS command
435 * @last_command: keeps track of last command on this chip. used
436 * for reading correct status
438 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
439 * ecc/non-ecc mode for the current nand flash
442 struct qcom_nand_host
{
443 struct nand_chip chip
;
444 struct list_head node
;
458 u32 cfg0_raw
, cfg1_raw
;
466 * This data type corresponds to the NAND controller properties which varies
467 * among different NAND controllers.
468 * @ecc_modes - ecc mode for NAND
469 * @is_bam - whether NAND controller is using BAM
470 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
472 struct qcom_nandc_props
{
475 u32 dev_cmd_reg_start
;
478 /* Frees the BAM transaction memory */
479 static void free_bam_transaction(struct qcom_nand_controller
*nandc
)
481 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
483 devm_kfree(nandc
->dev
, bam_txn
);
486 /* Allocates and Initializes the BAM transaction */
487 static struct bam_transaction
*
488 alloc_bam_transaction(struct qcom_nand_controller
*nandc
)
490 struct bam_transaction
*bam_txn
;
492 unsigned int num_cw
= nandc
->max_cwperpage
;
496 sizeof(*bam_txn
) + num_cw
*
497 ((sizeof(*bam_txn
->bam_ce
) * QPIC_PER_CW_CMD_ELEMENTS
) +
498 (sizeof(*bam_txn
->cmd_sgl
) * QPIC_PER_CW_CMD_SGL
) +
499 (sizeof(*bam_txn
->data_sgl
) * QPIC_PER_CW_DATA_SGL
));
501 bam_txn_buf
= devm_kzalloc(nandc
->dev
, bam_txn_size
, GFP_KERNEL
);
505 bam_txn
= bam_txn_buf
;
506 bam_txn_buf
+= sizeof(*bam_txn
);
508 bam_txn
->bam_ce
= bam_txn_buf
;
510 sizeof(*bam_txn
->bam_ce
) * QPIC_PER_CW_CMD_ELEMENTS
* num_cw
;
512 bam_txn
->cmd_sgl
= bam_txn_buf
;
514 sizeof(*bam_txn
->cmd_sgl
) * QPIC_PER_CW_CMD_SGL
* num_cw
;
516 bam_txn
->data_sgl
= bam_txn_buf
;
518 init_completion(&bam_txn
->txn_done
);
523 /* Clears the BAM transaction indexes */
524 static void clear_bam_transaction(struct qcom_nand_controller
*nandc
)
526 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
528 if (!nandc
->props
->is_bam
)
531 bam_txn
->bam_ce_pos
= 0;
532 bam_txn
->bam_ce_start
= 0;
533 bam_txn
->cmd_sgl_pos
= 0;
534 bam_txn
->cmd_sgl_start
= 0;
535 bam_txn
->tx_sgl_pos
= 0;
536 bam_txn
->tx_sgl_start
= 0;
537 bam_txn
->rx_sgl_pos
= 0;
538 bam_txn
->rx_sgl_start
= 0;
539 bam_txn
->last_data_desc
= NULL
;
540 bam_txn
->wait_second_completion
= false;
542 sg_init_table(bam_txn
->cmd_sgl
, nandc
->max_cwperpage
*
543 QPIC_PER_CW_CMD_SGL
);
544 sg_init_table(bam_txn
->data_sgl
, nandc
->max_cwperpage
*
545 QPIC_PER_CW_DATA_SGL
);
547 reinit_completion(&bam_txn
->txn_done
);
550 /* Callback for DMA descriptor completion */
551 static void qpic_bam_dma_done(void *data
)
553 struct bam_transaction
*bam_txn
= data
;
556 * In case of data transfer with NAND, 2 callbacks will be generated.
557 * One for command channel and another one for data channel.
558 * If current transaction has data descriptors
559 * (i.e. wait_second_completion is true), then set this to false
560 * and wait for second DMA descriptor completion.
562 if (bam_txn
->wait_second_completion
)
563 bam_txn
->wait_second_completion
= false;
565 complete(&bam_txn
->txn_done
);
568 static inline struct qcom_nand_host
*to_qcom_nand_host(struct nand_chip
*chip
)
570 return container_of(chip
, struct qcom_nand_host
, chip
);
573 static inline struct qcom_nand_controller
*
574 get_qcom_nand_controller(struct nand_chip
*chip
)
576 return container_of(chip
->controller
, struct qcom_nand_controller
,
580 static inline u32
nandc_read(struct qcom_nand_controller
*nandc
, int offset
)
582 return ioread32(nandc
->base
+ offset
);
585 static inline void nandc_write(struct qcom_nand_controller
*nandc
, int offset
,
588 iowrite32(val
, nandc
->base
+ offset
);
591 static inline void nandc_read_buffer_sync(struct qcom_nand_controller
*nandc
,
594 if (!nandc
->props
->is_bam
)
598 dma_sync_single_for_cpu(nandc
->dev
, nandc
->reg_read_dma
,
600 sizeof(*nandc
->reg_read_buf
),
603 dma_sync_single_for_device(nandc
->dev
, nandc
->reg_read_dma
,
605 sizeof(*nandc
->reg_read_buf
),
609 static __le32
*offset_to_nandc_reg(struct nandc_regs
*regs
, int offset
)
618 case NAND_FLASH_CHIP_SELECT
:
619 return ®s
->chip_sel
;
622 case NAND_FLASH_STATUS
:
623 return ®s
->clrflashstatus
;
628 case NAND_DEV0_ECC_CFG
:
629 return ®s
->ecc_bch_cfg
;
630 case NAND_READ_STATUS
:
631 return ®s
->clrreadstatus
;
634 case NAND_DEV_CMD1_RESTORE
:
635 return ®s
->orig_cmd1
;
636 case NAND_DEV_CMD_VLD
:
638 case NAND_DEV_CMD_VLD_RESTORE
:
639 return ®s
->orig_vld
;
640 case NAND_EBI2_ECC_BUF_CFG
:
641 return ®s
->ecc_buf_cfg
;
642 case NAND_READ_LOCATION_0
:
643 return ®s
->read_location0
;
644 case NAND_READ_LOCATION_1
:
645 return ®s
->read_location1
;
646 case NAND_READ_LOCATION_2
:
647 return ®s
->read_location2
;
648 case NAND_READ_LOCATION_3
:
649 return ®s
->read_location3
;
655 static void nandc_set_reg(struct qcom_nand_controller
*nandc
, int offset
,
658 struct nandc_regs
*regs
= nandc
->regs
;
661 reg
= offset_to_nandc_reg(regs
, offset
);
664 *reg
= cpu_to_le32(val
);
667 /* helper to configure address register values */
668 static void set_address(struct qcom_nand_host
*host
, u16 column
, int page
)
670 struct nand_chip
*chip
= &host
->chip
;
671 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
673 if (chip
->options
& NAND_BUSWIDTH_16
)
676 nandc_set_reg(nandc
, NAND_ADDR0
, page
<< 16 | column
);
677 nandc_set_reg(nandc
, NAND_ADDR1
, page
>> 16 & 0xff);
681 * update_rw_regs: set up read/write register values, these will be
682 * written to the NAND controller registers via DMA
684 * @num_cw: number of steps for the read/write operation
685 * @read: read or write operation
687 static void update_rw_regs(struct qcom_nand_host
*host
, int num_cw
, bool read
)
689 struct nand_chip
*chip
= &host
->chip
;
690 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
691 u32 cmd
, cfg0
, cfg1
, ecc_bch_cfg
;
695 cmd
= OP_PAGE_READ_WITH_ECC
| PAGE_ACC
| LAST_PAGE
;
697 cmd
= OP_PAGE_READ
| PAGE_ACC
| LAST_PAGE
;
699 cmd
= OP_PROGRAM_PAGE
| PAGE_ACC
| LAST_PAGE
;
703 cfg0
= (host
->cfg0
& ~(7U << CW_PER_PAGE
)) |
704 (num_cw
- 1) << CW_PER_PAGE
;
707 ecc_bch_cfg
= host
->ecc_bch_cfg
;
709 cfg0
= (host
->cfg0_raw
& ~(7U << CW_PER_PAGE
)) |
710 (num_cw
- 1) << CW_PER_PAGE
;
712 cfg1
= host
->cfg1_raw
;
713 ecc_bch_cfg
= 1 << ECC_CFG_ECC_DISABLE
;
716 nandc_set_reg(nandc
, NAND_FLASH_CMD
, cmd
);
717 nandc_set_reg(nandc
, NAND_DEV0_CFG0
, cfg0
);
718 nandc_set_reg(nandc
, NAND_DEV0_CFG1
, cfg1
);
719 nandc_set_reg(nandc
, NAND_DEV0_ECC_CFG
, ecc_bch_cfg
);
720 nandc_set_reg(nandc
, NAND_EBI2_ECC_BUF_CFG
, host
->ecc_buf_cfg
);
721 nandc_set_reg(nandc
, NAND_FLASH_STATUS
, host
->clrflashstatus
);
722 nandc_set_reg(nandc
, NAND_READ_STATUS
, host
->clrreadstatus
);
723 nandc_set_reg(nandc
, NAND_EXEC_CMD
, 1);
726 nandc_set_read_loc(nandc
, 0, 0, host
->use_ecc
?
727 host
->cw_data
: host
->cw_size
, 1);
731 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
732 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
733 * which will be submitted to DMA engine.
735 static int prepare_bam_async_desc(struct qcom_nand_controller
*nandc
,
736 struct dma_chan
*chan
,
739 struct desc_info
*desc
;
740 struct scatterlist
*sgl
;
741 unsigned int sgl_cnt
;
743 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
744 enum dma_transfer_direction dir_eng
;
745 struct dma_async_tx_descriptor
*dma_desc
;
747 desc
= kzalloc(sizeof(*desc
), GFP_KERNEL
);
751 if (chan
== nandc
->cmd_chan
) {
752 sgl
= &bam_txn
->cmd_sgl
[bam_txn
->cmd_sgl_start
];
753 sgl_cnt
= bam_txn
->cmd_sgl_pos
- bam_txn
->cmd_sgl_start
;
754 bam_txn
->cmd_sgl_start
= bam_txn
->cmd_sgl_pos
;
755 dir_eng
= DMA_MEM_TO_DEV
;
756 desc
->dir
= DMA_TO_DEVICE
;
757 } else if (chan
== nandc
->tx_chan
) {
758 sgl
= &bam_txn
->data_sgl
[bam_txn
->tx_sgl_start
];
759 sgl_cnt
= bam_txn
->tx_sgl_pos
- bam_txn
->tx_sgl_start
;
760 bam_txn
->tx_sgl_start
= bam_txn
->tx_sgl_pos
;
761 dir_eng
= DMA_MEM_TO_DEV
;
762 desc
->dir
= DMA_TO_DEVICE
;
764 sgl
= &bam_txn
->data_sgl
[bam_txn
->rx_sgl_start
];
765 sgl_cnt
= bam_txn
->rx_sgl_pos
- bam_txn
->rx_sgl_start
;
766 bam_txn
->rx_sgl_start
= bam_txn
->rx_sgl_pos
;
767 dir_eng
= DMA_DEV_TO_MEM
;
768 desc
->dir
= DMA_FROM_DEVICE
;
771 sg_mark_end(sgl
+ sgl_cnt
- 1);
772 ret
= dma_map_sg(nandc
->dev
, sgl
, sgl_cnt
, desc
->dir
);
774 dev_err(nandc
->dev
, "failure in mapping desc\n");
779 desc
->sgl_cnt
= sgl_cnt
;
782 dma_desc
= dmaengine_prep_slave_sg(chan
, sgl
, sgl_cnt
, dir_eng
,
786 dev_err(nandc
->dev
, "failure in prep desc\n");
787 dma_unmap_sg(nandc
->dev
, sgl
, sgl_cnt
, desc
->dir
);
792 desc
->dma_desc
= dma_desc
;
794 /* update last data/command descriptor */
795 if (chan
== nandc
->cmd_chan
)
796 bam_txn
->last_cmd_desc
= dma_desc
;
798 bam_txn
->last_data_desc
= dma_desc
;
800 list_add_tail(&desc
->node
, &nandc
->desc_list
);
806 * Prepares the command descriptor for BAM DMA which will be used for NAND
807 * register reads and writes. The command descriptor requires the command
808 * to be formed in command element type so this function uses the command
809 * element from bam transaction ce array and fills the same with required
810 * data. A single SGL can contain multiple command elements so
811 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
812 * after the current command element.
814 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller
*nandc
, bool read
,
815 int reg_off
, const void *vaddr
,
816 int size
, unsigned int flags
)
820 struct bam_cmd_element
*bam_ce_buffer
;
821 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
823 bam_ce_buffer
= &bam_txn
->bam_ce
[bam_txn
->bam_ce_pos
];
825 /* fill the command desc */
826 for (i
= 0; i
< size
; i
++) {
828 bam_prep_ce(&bam_ce_buffer
[i
],
829 nandc_reg_phys(nandc
, reg_off
+ 4 * i
),
831 reg_buf_dma_addr(nandc
,
832 (__le32
*)vaddr
+ i
));
834 bam_prep_ce_le32(&bam_ce_buffer
[i
],
835 nandc_reg_phys(nandc
, reg_off
+ 4 * i
),
837 *((__le32
*)vaddr
+ i
));
840 bam_txn
->bam_ce_pos
+= size
;
842 /* use the separate sgl after this command */
843 if (flags
& NAND_BAM_NEXT_SGL
) {
844 bam_ce_buffer
= &bam_txn
->bam_ce
[bam_txn
->bam_ce_start
];
845 bam_ce_size
= (bam_txn
->bam_ce_pos
-
846 bam_txn
->bam_ce_start
) *
847 sizeof(struct bam_cmd_element
);
848 sg_set_buf(&bam_txn
->cmd_sgl
[bam_txn
->cmd_sgl_pos
],
849 bam_ce_buffer
, bam_ce_size
);
850 bam_txn
->cmd_sgl_pos
++;
851 bam_txn
->bam_ce_start
= bam_txn
->bam_ce_pos
;
853 if (flags
& NAND_BAM_NWD
) {
854 ret
= prepare_bam_async_desc(nandc
, nandc
->cmd_chan
,
866 * Prepares the data descriptor for BAM DMA which will be used for NAND
867 * data reads and writes.
869 static int prep_bam_dma_desc_data(struct qcom_nand_controller
*nandc
, bool read
,
871 int size
, unsigned int flags
)
874 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
877 sg_set_buf(&bam_txn
->data_sgl
[bam_txn
->rx_sgl_pos
],
879 bam_txn
->rx_sgl_pos
++;
881 sg_set_buf(&bam_txn
->data_sgl
[bam_txn
->tx_sgl_pos
],
883 bam_txn
->tx_sgl_pos
++;
886 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
887 * is not set, form the DMA descriptor
889 if (!(flags
& NAND_BAM_NO_EOT
)) {
890 ret
= prepare_bam_async_desc(nandc
, nandc
->tx_chan
,
900 static int prep_adm_dma_desc(struct qcom_nand_controller
*nandc
, bool read
,
901 int reg_off
, const void *vaddr
, int size
,
904 struct desc_info
*desc
;
905 struct dma_async_tx_descriptor
*dma_desc
;
906 struct scatterlist
*sgl
;
907 struct dma_slave_config slave_conf
;
908 enum dma_transfer_direction dir_eng
;
911 desc
= kzalloc(sizeof(*desc
), GFP_KERNEL
);
915 sgl
= &desc
->adm_sgl
;
917 sg_init_one(sgl
, vaddr
, size
);
920 dir_eng
= DMA_DEV_TO_MEM
;
921 desc
->dir
= DMA_FROM_DEVICE
;
923 dir_eng
= DMA_MEM_TO_DEV
;
924 desc
->dir
= DMA_TO_DEVICE
;
927 ret
= dma_map_sg(nandc
->dev
, sgl
, 1, desc
->dir
);
933 memset(&slave_conf
, 0x00, sizeof(slave_conf
));
935 slave_conf
.device_fc
= flow_control
;
937 slave_conf
.src_maxburst
= 16;
938 slave_conf
.src_addr
= nandc
->base_dma
+ reg_off
;
939 slave_conf
.slave_id
= nandc
->data_crci
;
941 slave_conf
.dst_maxburst
= 16;
942 slave_conf
.dst_addr
= nandc
->base_dma
+ reg_off
;
943 slave_conf
.slave_id
= nandc
->cmd_crci
;
946 ret
= dmaengine_slave_config(nandc
->chan
, &slave_conf
);
948 dev_err(nandc
->dev
, "failed to configure dma channel\n");
952 dma_desc
= dmaengine_prep_slave_sg(nandc
->chan
, sgl
, 1, dir_eng
, 0);
954 dev_err(nandc
->dev
, "failed to prepare desc\n");
959 desc
->dma_desc
= dma_desc
;
961 list_add_tail(&desc
->node
, &nandc
->desc_list
);
971 * read_reg_dma: prepares a descriptor to read a given number of
972 * contiguous registers to the reg_read_buf pointer
974 * @first: offset of the first register in the contiguous block
975 * @num_regs: number of registers to read
976 * @flags: flags to control DMA descriptor preparation
978 static int read_reg_dma(struct qcom_nand_controller
*nandc
, int first
,
979 int num_regs
, unsigned int flags
)
981 bool flow_control
= false;
984 vaddr
= nandc
->reg_read_buf
+ nandc
->reg_read_pos
;
985 nandc
->reg_read_pos
+= num_regs
;
987 if (first
== NAND_DEV_CMD_VLD
|| first
== NAND_DEV_CMD1
)
988 first
= dev_cmd_reg_addr(nandc
, first
);
990 if (nandc
->props
->is_bam
)
991 return prep_bam_dma_desc_cmd(nandc
, true, first
, vaddr
,
994 if (first
== NAND_READ_ID
|| first
== NAND_FLASH_STATUS
)
997 return prep_adm_dma_desc(nandc
, true, first
, vaddr
,
998 num_regs
* sizeof(u32
), flow_control
);
1002 * write_reg_dma: prepares a descriptor to write a given number of
1003 * contiguous registers
1005 * @first: offset of the first register in the contiguous block
1006 * @num_regs: number of registers to write
1007 * @flags: flags to control DMA descriptor preparation
1009 static int write_reg_dma(struct qcom_nand_controller
*nandc
, int first
,
1010 int num_regs
, unsigned int flags
)
1012 bool flow_control
= false;
1013 struct nandc_regs
*regs
= nandc
->regs
;
1016 vaddr
= offset_to_nandc_reg(regs
, first
);
1018 if (first
== NAND_ERASED_CW_DETECT_CFG
) {
1019 if (flags
& NAND_ERASED_CW_SET
)
1020 vaddr
= ®s
->erased_cw_detect_cfg_set
;
1022 vaddr
= ®s
->erased_cw_detect_cfg_clr
;
1025 if (first
== NAND_EXEC_CMD
)
1026 flags
|= NAND_BAM_NWD
;
1028 if (first
== NAND_DEV_CMD1_RESTORE
|| first
== NAND_DEV_CMD1
)
1029 first
= dev_cmd_reg_addr(nandc
, NAND_DEV_CMD1
);
1031 if (first
== NAND_DEV_CMD_VLD_RESTORE
|| first
== NAND_DEV_CMD_VLD
)
1032 first
= dev_cmd_reg_addr(nandc
, NAND_DEV_CMD_VLD
);
1034 if (nandc
->props
->is_bam
)
1035 return prep_bam_dma_desc_cmd(nandc
, false, first
, vaddr
,
1038 if (first
== NAND_FLASH_CMD
)
1039 flow_control
= true;
1041 return prep_adm_dma_desc(nandc
, false, first
, vaddr
,
1042 num_regs
* sizeof(u32
), flow_control
);
1046 * read_data_dma: prepares a DMA descriptor to transfer data from the
1047 * controller's internal buffer to the buffer 'vaddr'
1049 * @reg_off: offset within the controller's data buffer
1050 * @vaddr: virtual address of the buffer we want to write to
1051 * @size: DMA transaction size in bytes
1052 * @flags: flags to control DMA descriptor preparation
1054 static int read_data_dma(struct qcom_nand_controller
*nandc
, int reg_off
,
1055 const u8
*vaddr
, int size
, unsigned int flags
)
1057 if (nandc
->props
->is_bam
)
1058 return prep_bam_dma_desc_data(nandc
, true, vaddr
, size
, flags
);
1060 return prep_adm_dma_desc(nandc
, true, reg_off
, vaddr
, size
, false);
1064 * write_data_dma: prepares a DMA descriptor to transfer data from
1065 * 'vaddr' to the controller's internal buffer
1067 * @reg_off: offset within the controller's data buffer
1068 * @vaddr: virtual address of the buffer we want to read from
1069 * @size: DMA transaction size in bytes
1070 * @flags: flags to control DMA descriptor preparation
1072 static int write_data_dma(struct qcom_nand_controller
*nandc
, int reg_off
,
1073 const u8
*vaddr
, int size
, unsigned int flags
)
1075 if (nandc
->props
->is_bam
)
1076 return prep_bam_dma_desc_data(nandc
, false, vaddr
, size
, flags
);
1078 return prep_adm_dma_desc(nandc
, false, reg_off
, vaddr
, size
, false);
1082 * Helper to prepare DMA descriptors for configuring registers
1083 * before reading a NAND page.
1085 static void config_nand_page_read(struct qcom_nand_controller
*nandc
)
1087 write_reg_dma(nandc
, NAND_ADDR0
, 2, 0);
1088 write_reg_dma(nandc
, NAND_DEV0_CFG0
, 3, 0);
1089 write_reg_dma(nandc
, NAND_EBI2_ECC_BUF_CFG
, 1, 0);
1090 write_reg_dma(nandc
, NAND_ERASED_CW_DETECT_CFG
, 1, 0);
1091 write_reg_dma(nandc
, NAND_ERASED_CW_DETECT_CFG
, 1,
1092 NAND_ERASED_CW_SET
| NAND_BAM_NEXT_SGL
);
1096 * Helper to prepare DMA descriptors for configuring registers
1097 * before reading each codeword in NAND page.
1100 config_nand_cw_read(struct qcom_nand_controller
*nandc
, bool use_ecc
)
1102 if (nandc
->props
->is_bam
)
1103 write_reg_dma(nandc
, NAND_READ_LOCATION_0
, 4,
1106 write_reg_dma(nandc
, NAND_FLASH_CMD
, 1, NAND_BAM_NEXT_SGL
);
1107 write_reg_dma(nandc
, NAND_EXEC_CMD
, 1, NAND_BAM_NEXT_SGL
);
1110 read_reg_dma(nandc
, NAND_FLASH_STATUS
, 2, 0);
1111 read_reg_dma(nandc
, NAND_ERASED_CW_DETECT_STATUS
, 1,
1114 read_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1119 * Helper to prepare dma descriptors to configure registers needed for reading a
1120 * single codeword in page
1123 config_nand_single_cw_page_read(struct qcom_nand_controller
*nandc
,
1126 config_nand_page_read(nandc
);
1127 config_nand_cw_read(nandc
, use_ecc
);
1131 * Helper to prepare DMA descriptors used to configure registers needed for
1132 * before writing a NAND page.
1134 static void config_nand_page_write(struct qcom_nand_controller
*nandc
)
1136 write_reg_dma(nandc
, NAND_ADDR0
, 2, 0);
1137 write_reg_dma(nandc
, NAND_DEV0_CFG0
, 3, 0);
1138 write_reg_dma(nandc
, NAND_EBI2_ECC_BUF_CFG
, 1,
1143 * Helper to prepare DMA descriptors for configuring registers
1144 * before writing each codeword in NAND page.
1146 static void config_nand_cw_write(struct qcom_nand_controller
*nandc
)
1148 write_reg_dma(nandc
, NAND_FLASH_CMD
, 1, NAND_BAM_NEXT_SGL
);
1149 write_reg_dma(nandc
, NAND_EXEC_CMD
, 1, NAND_BAM_NEXT_SGL
);
1151 read_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1153 write_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, 0);
1154 write_reg_dma(nandc
, NAND_READ_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1158 * the following functions are used within chip->legacy.cmdfunc() to
1159 * perform different NAND_CMD_* commands
1162 /* sets up descriptors for NAND_CMD_PARAM */
1163 static int nandc_param(struct qcom_nand_host
*host
)
1165 struct nand_chip
*chip
= &host
->chip
;
1166 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1169 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1170 * in use. we configure the controller to perform a raw read of 512
1171 * bytes to read onfi params
1173 nandc_set_reg(nandc
, NAND_FLASH_CMD
, OP_PAGE_READ
| PAGE_ACC
| LAST_PAGE
);
1174 nandc_set_reg(nandc
, NAND_ADDR0
, 0);
1175 nandc_set_reg(nandc
, NAND_ADDR1
, 0);
1176 nandc_set_reg(nandc
, NAND_DEV0_CFG0
, 0 << CW_PER_PAGE
1177 | 512 << UD_SIZE_BYTES
1178 | 5 << NUM_ADDR_CYCLES
1179 | 0 << SPARE_SIZE_BYTES
);
1180 nandc_set_reg(nandc
, NAND_DEV0_CFG1
, 7 << NAND_RECOVERY_CYCLES
1181 | 0 << CS_ACTIVE_BSY
1182 | 17 << BAD_BLOCK_BYTE_NUM
1183 | 1 << BAD_BLOCK_IN_SPARE_AREA
1184 | 2 << WR_RD_BSY_GAP
1186 | 1 << DEV0_CFG1_ECC_DISABLE
);
1187 nandc_set_reg(nandc
, NAND_EBI2_ECC_BUF_CFG
, 1 << ECC_CFG_ECC_DISABLE
);
1189 /* configure CMD1 and VLD for ONFI param probing */
1190 nandc_set_reg(nandc
, NAND_DEV_CMD_VLD
,
1191 (nandc
->vld
& ~READ_START_VLD
));
1192 nandc_set_reg(nandc
, NAND_DEV_CMD1
,
1193 (nandc
->cmd1
& ~(0xFF << READ_ADDR
))
1194 | NAND_CMD_PARAM
<< READ_ADDR
);
1196 nandc_set_reg(nandc
, NAND_EXEC_CMD
, 1);
1198 nandc_set_reg(nandc
, NAND_DEV_CMD1_RESTORE
, nandc
->cmd1
);
1199 nandc_set_reg(nandc
, NAND_DEV_CMD_VLD_RESTORE
, nandc
->vld
);
1200 nandc_set_read_loc(nandc
, 0, 0, 512, 1);
1202 write_reg_dma(nandc
, NAND_DEV_CMD_VLD
, 1, 0);
1203 write_reg_dma(nandc
, NAND_DEV_CMD1
, 1, NAND_BAM_NEXT_SGL
);
1205 nandc
->buf_count
= 512;
1206 memset(nandc
->data_buffer
, 0xff, nandc
->buf_count
);
1208 config_nand_single_cw_page_read(nandc
, false);
1210 read_data_dma(nandc
, FLASH_BUF_ACC
, nandc
->data_buffer
,
1211 nandc
->buf_count
, 0);
1213 /* restore CMD1 and VLD regs */
1214 write_reg_dma(nandc
, NAND_DEV_CMD1_RESTORE
, 1, 0);
1215 write_reg_dma(nandc
, NAND_DEV_CMD_VLD_RESTORE
, 1, NAND_BAM_NEXT_SGL
);
1220 /* sets up descriptors for NAND_CMD_ERASE1 */
1221 static int erase_block(struct qcom_nand_host
*host
, int page_addr
)
1223 struct nand_chip
*chip
= &host
->chip
;
1224 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1226 nandc_set_reg(nandc
, NAND_FLASH_CMD
,
1227 OP_BLOCK_ERASE
| PAGE_ACC
| LAST_PAGE
);
1228 nandc_set_reg(nandc
, NAND_ADDR0
, page_addr
);
1229 nandc_set_reg(nandc
, NAND_ADDR1
, 0);
1230 nandc_set_reg(nandc
, NAND_DEV0_CFG0
,
1231 host
->cfg0_raw
& ~(7 << CW_PER_PAGE
));
1232 nandc_set_reg(nandc
, NAND_DEV0_CFG1
, host
->cfg1_raw
);
1233 nandc_set_reg(nandc
, NAND_EXEC_CMD
, 1);
1234 nandc_set_reg(nandc
, NAND_FLASH_STATUS
, host
->clrflashstatus
);
1235 nandc_set_reg(nandc
, NAND_READ_STATUS
, host
->clrreadstatus
);
1237 write_reg_dma(nandc
, NAND_FLASH_CMD
, 3, NAND_BAM_NEXT_SGL
);
1238 write_reg_dma(nandc
, NAND_DEV0_CFG0
, 2, NAND_BAM_NEXT_SGL
);
1239 write_reg_dma(nandc
, NAND_EXEC_CMD
, 1, NAND_BAM_NEXT_SGL
);
1241 read_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1243 write_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, 0);
1244 write_reg_dma(nandc
, NAND_READ_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1249 /* sets up descriptors for NAND_CMD_READID */
1250 static int read_id(struct qcom_nand_host
*host
, int column
)
1252 struct nand_chip
*chip
= &host
->chip
;
1253 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1258 nandc_set_reg(nandc
, NAND_FLASH_CMD
, OP_FETCH_ID
);
1259 nandc_set_reg(nandc
, NAND_ADDR0
, column
);
1260 nandc_set_reg(nandc
, NAND_ADDR1
, 0);
1261 nandc_set_reg(nandc
, NAND_FLASH_CHIP_SELECT
,
1262 nandc
->props
->is_bam
? 0 : DM_EN
);
1263 nandc_set_reg(nandc
, NAND_EXEC_CMD
, 1);
1265 write_reg_dma(nandc
, NAND_FLASH_CMD
, 4, NAND_BAM_NEXT_SGL
);
1266 write_reg_dma(nandc
, NAND_EXEC_CMD
, 1, NAND_BAM_NEXT_SGL
);
1268 read_reg_dma(nandc
, NAND_READ_ID
, 1, NAND_BAM_NEXT_SGL
);
1273 /* sets up descriptors for NAND_CMD_RESET */
1274 static int reset(struct qcom_nand_host
*host
)
1276 struct nand_chip
*chip
= &host
->chip
;
1277 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1279 nandc_set_reg(nandc
, NAND_FLASH_CMD
, OP_RESET_DEVICE
);
1280 nandc_set_reg(nandc
, NAND_EXEC_CMD
, 1);
1282 write_reg_dma(nandc
, NAND_FLASH_CMD
, 1, NAND_BAM_NEXT_SGL
);
1283 write_reg_dma(nandc
, NAND_EXEC_CMD
, 1, NAND_BAM_NEXT_SGL
);
1285 read_reg_dma(nandc
, NAND_FLASH_STATUS
, 1, NAND_BAM_NEXT_SGL
);
1290 /* helpers to submit/free our list of dma descriptors */
1291 static int submit_descs(struct qcom_nand_controller
*nandc
)
1293 struct desc_info
*desc
;
1294 dma_cookie_t cookie
= 0;
1295 struct bam_transaction
*bam_txn
= nandc
->bam_txn
;
1298 if (nandc
->props
->is_bam
) {
1299 if (bam_txn
->rx_sgl_pos
> bam_txn
->rx_sgl_start
) {
1300 r
= prepare_bam_async_desc(nandc
, nandc
->rx_chan
, 0);
1305 if (bam_txn
->tx_sgl_pos
> bam_txn
->tx_sgl_start
) {
1306 r
= prepare_bam_async_desc(nandc
, nandc
->tx_chan
,
1307 DMA_PREP_INTERRUPT
);
1312 if (bam_txn
->cmd_sgl_pos
> bam_txn
->cmd_sgl_start
) {
1313 r
= prepare_bam_async_desc(nandc
, nandc
->cmd_chan
,
1320 list_for_each_entry(desc
, &nandc
->desc_list
, node
)
1321 cookie
= dmaengine_submit(desc
->dma_desc
);
1323 if (nandc
->props
->is_bam
) {
1324 bam_txn
->last_cmd_desc
->callback
= qpic_bam_dma_done
;
1325 bam_txn
->last_cmd_desc
->callback_param
= bam_txn
;
1326 if (bam_txn
->last_data_desc
) {
1327 bam_txn
->last_data_desc
->callback
= qpic_bam_dma_done
;
1328 bam_txn
->last_data_desc
->callback_param
= bam_txn
;
1329 bam_txn
->wait_second_completion
= true;
1332 dma_async_issue_pending(nandc
->tx_chan
);
1333 dma_async_issue_pending(nandc
->rx_chan
);
1334 dma_async_issue_pending(nandc
->cmd_chan
);
1336 if (!wait_for_completion_timeout(&bam_txn
->txn_done
,
1337 QPIC_NAND_COMPLETION_TIMEOUT
))
1340 if (dma_sync_wait(nandc
->chan
, cookie
) != DMA_COMPLETE
)
1347 static void free_descs(struct qcom_nand_controller
*nandc
)
1349 struct desc_info
*desc
, *n
;
1351 list_for_each_entry_safe(desc
, n
, &nandc
->desc_list
, node
) {
1352 list_del(&desc
->node
);
1354 if (nandc
->props
->is_bam
)
1355 dma_unmap_sg(nandc
->dev
, desc
->bam_sgl
,
1356 desc
->sgl_cnt
, desc
->dir
);
1358 dma_unmap_sg(nandc
->dev
, &desc
->adm_sgl
, 1,
1365 /* reset the register read buffer for next NAND operation */
1366 static void clear_read_regs(struct qcom_nand_controller
*nandc
)
1368 nandc
->reg_read_pos
= 0;
1369 nandc_read_buffer_sync(nandc
, false);
1372 static void pre_command(struct qcom_nand_host
*host
, int command
)
1374 struct nand_chip
*chip
= &host
->chip
;
1375 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1377 nandc
->buf_count
= 0;
1378 nandc
->buf_start
= 0;
1379 host
->use_ecc
= false;
1380 host
->last_command
= command
;
1382 clear_read_regs(nandc
);
1384 if (command
== NAND_CMD_RESET
|| command
== NAND_CMD_READID
||
1385 command
== NAND_CMD_PARAM
|| command
== NAND_CMD_ERASE1
)
1386 clear_bam_transaction(nandc
);
1390 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1391 * privately maintained status byte, this status byte can be read after
1392 * NAND_CMD_STATUS is called
1394 static void parse_erase_write_errors(struct qcom_nand_host
*host
, int command
)
1396 struct nand_chip
*chip
= &host
->chip
;
1397 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1398 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1402 num_cw
= command
== NAND_CMD_PAGEPROG
? ecc
->steps
: 1;
1403 nandc_read_buffer_sync(nandc
, true);
1405 for (i
= 0; i
< num_cw
; i
++) {
1406 u32 flash_status
= le32_to_cpu(nandc
->reg_read_buf
[i
]);
1408 if (flash_status
& FS_MPU_ERR
)
1409 host
->status
&= ~NAND_STATUS_WP
;
1411 if (flash_status
& FS_OP_ERR
|| (i
== (num_cw
- 1) &&
1413 FS_DEVICE_STS_ERR
)))
1414 host
->status
|= NAND_STATUS_FAIL
;
1418 static void post_command(struct qcom_nand_host
*host
, int command
)
1420 struct nand_chip
*chip
= &host
->chip
;
1421 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1424 case NAND_CMD_READID
:
1425 nandc_read_buffer_sync(nandc
, true);
1426 memcpy(nandc
->data_buffer
, nandc
->reg_read_buf
,
1429 case NAND_CMD_PAGEPROG
:
1430 case NAND_CMD_ERASE1
:
1431 parse_erase_write_errors(host
, command
);
1439 * Implements chip->legacy.cmdfunc. It's only used for a limited set of
1440 * commands. The rest of the commands wouldn't be called by upper layers.
1441 * For example, NAND_CMD_READOOB would never be called because we have our own
1442 * versions of read_oob ops for nand_ecc_ctrl.
1444 static void qcom_nandc_command(struct nand_chip
*chip
, unsigned int command
,
1445 int column
, int page_addr
)
1447 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
1448 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1449 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1453 pre_command(host
, command
);
1456 case NAND_CMD_RESET
:
1461 case NAND_CMD_READID
:
1462 nandc
->buf_count
= 4;
1463 ret
= read_id(host
, column
);
1467 case NAND_CMD_PARAM
:
1468 ret
= nandc_param(host
);
1472 case NAND_CMD_ERASE1
:
1473 ret
= erase_block(host
, page_addr
);
1477 case NAND_CMD_READ0
:
1478 /* we read the entire page for now */
1479 WARN_ON(column
!= 0);
1481 host
->use_ecc
= true;
1482 set_address(host
, 0, page_addr
);
1483 update_rw_regs(host
, ecc
->steps
, true);
1486 case NAND_CMD_SEQIN
:
1487 WARN_ON(column
!= 0);
1488 set_address(host
, 0, page_addr
);
1491 case NAND_CMD_PAGEPROG
:
1492 case NAND_CMD_STATUS
:
1499 dev_err(nandc
->dev
, "failure executing command %d\n",
1506 ret
= submit_descs(nandc
);
1509 "failure submitting descs for command %d\n",
1515 post_command(host
, command
);
1519 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1520 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1522 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1523 * but it notifies that it is an erased CW by placing special characters at
1524 * certain offsets in the buffer.
1526 * verify if the page is erased or not, and fix up the page for RS ECC by
1527 * replacing the special characters with 0xff.
1529 static bool erased_chunk_check_and_fixup(u8
*data_buf
, int data_len
)
1534 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1535 * is erased by looking for 0x54s at offsets 3 and 175 from the
1536 * beginning of each codeword
1539 empty1
= data_buf
[3];
1540 empty2
= data_buf
[175];
1543 * if the erased codework markers, if they exist override them with
1546 if ((empty1
== 0x54 && empty2
== 0xff) ||
1547 (empty1
== 0xff && empty2
== 0x54)) {
1549 data_buf
[175] = 0xff;
1553 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1554 * restore the original values at the special offsets
1556 if (memchr_inv(data_buf
, 0xff, data_len
)) {
1557 data_buf
[3] = empty1
;
1558 data_buf
[175] = empty2
;
1572 /* reads back FLASH_STATUS register set by the controller */
1573 static int check_flash_errors(struct qcom_nand_host
*host
, int cw_cnt
)
1575 struct nand_chip
*chip
= &host
->chip
;
1576 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1579 for (i
= 0; i
< cw_cnt
; i
++) {
1580 u32 flash
= le32_to_cpu(nandc
->reg_read_buf
[i
]);
1582 if (flash
& (FS_OP_ERR
| FS_MPU_ERR
))
1589 /* performs raw read for one codeword */
1591 qcom_nandc_read_cw_raw(struct mtd_info
*mtd
, struct nand_chip
*chip
,
1592 u8
*data_buf
, u8
*oob_buf
, int page
, int cw
)
1594 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
1595 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1596 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1597 int data_size1
, data_size2
, oob_size1
, oob_size2
;
1598 int ret
, reg_off
= FLASH_BUF_ACC
, read_loc
= 0;
1600 nand_read_page_op(chip
, page
, 0, NULL
, 0);
1601 host
->use_ecc
= false;
1603 clear_bam_transaction(nandc
);
1604 set_address(host
, host
->cw_size
* cw
, page
);
1605 update_rw_regs(host
, 1, true);
1606 config_nand_page_read(nandc
);
1608 data_size1
= mtd
->writesize
- host
->cw_size
* (ecc
->steps
- 1);
1609 oob_size1
= host
->bbm_size
;
1611 if (cw
== (ecc
->steps
- 1)) {
1612 data_size2
= ecc
->size
- data_size1
-
1613 ((ecc
->steps
- 1) * 4);
1614 oob_size2
= (ecc
->steps
* 4) + host
->ecc_bytes_hw
+
1617 data_size2
= host
->cw_data
- data_size1
;
1618 oob_size2
= host
->ecc_bytes_hw
+ host
->spare_bytes
;
1621 if (nandc
->props
->is_bam
) {
1622 nandc_set_read_loc(nandc
, 0, read_loc
, data_size1
, 0);
1623 read_loc
+= data_size1
;
1625 nandc_set_read_loc(nandc
, 1, read_loc
, oob_size1
, 0);
1626 read_loc
+= oob_size1
;
1628 nandc_set_read_loc(nandc
, 2, read_loc
, data_size2
, 0);
1629 read_loc
+= data_size2
;
1631 nandc_set_read_loc(nandc
, 3, read_loc
, oob_size2
, 1);
1634 config_nand_cw_read(nandc
, false);
1636 read_data_dma(nandc
, reg_off
, data_buf
, data_size1
, 0);
1637 reg_off
+= data_size1
;
1639 read_data_dma(nandc
, reg_off
, oob_buf
, oob_size1
, 0);
1640 reg_off
+= oob_size1
;
1642 read_data_dma(nandc
, reg_off
, data_buf
+ data_size1
, data_size2
, 0);
1643 reg_off
+= data_size2
;
1645 read_data_dma(nandc
, reg_off
, oob_buf
+ oob_size1
, oob_size2
, 0);
1647 ret
= submit_descs(nandc
);
1650 dev_err(nandc
->dev
, "failure to read raw cw %d\n", cw
);
1654 return check_flash_errors(host
, 1);
1658 * Bitflips can happen in erased codewords also so this function counts the
1659 * number of 0 in each CW for which ECC engine returns the uncorrectable
1660 * error. The page will be assumed as erased if this count is less than or
1661 * equal to the ecc->strength for each CW.
1663 * 1. Both DATA and OOB need to be checked for number of 0. The
1664 * top-level API can be called with only data buf or OOB buf so use
1665 * chip->data_buf if data buf is null and chip->oob_poi if oob buf
1666 * is null for copying the raw bytes.
1667 * 2. Perform raw read for all the CW which has uncorrectable errors.
1668 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1669 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1670 * the number of bitflips in this area.
1673 check_for_erased_page(struct qcom_nand_host
*host
, u8
*data_buf
,
1674 u8
*oob_buf
, unsigned long uncorrectable_cws
,
1675 int page
, unsigned int max_bitflips
)
1677 struct nand_chip
*chip
= &host
->chip
;
1678 struct mtd_info
*mtd
= nand_to_mtd(chip
);
1679 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1680 u8
*cw_data_buf
, *cw_oob_buf
;
1681 int cw
, data_size
, oob_size
, ret
= 0;
1684 data_buf
= nand_get_data_buf(chip
);
1687 nand_get_data_buf(chip
);
1688 oob_buf
= chip
->oob_poi
;
1691 for_each_set_bit(cw
, &uncorrectable_cws
, ecc
->steps
) {
1692 if (cw
== (ecc
->steps
- 1)) {
1693 data_size
= ecc
->size
- ((ecc
->steps
- 1) * 4);
1694 oob_size
= (ecc
->steps
* 4) + host
->ecc_bytes_hw
;
1696 data_size
= host
->cw_data
;
1697 oob_size
= host
->ecc_bytes_hw
;
1700 /* determine starting buffer address for current CW */
1701 cw_data_buf
= data_buf
+ (cw
* host
->cw_data
);
1702 cw_oob_buf
= oob_buf
+ (cw
* ecc
->bytes
);
1704 ret
= qcom_nandc_read_cw_raw(mtd
, chip
, cw_data_buf
,
1705 cw_oob_buf
, page
, cw
);
1710 * make sure it isn't an erased page reported
1711 * as not-erased by HW because of a few bitflips
1713 ret
= nand_check_erased_ecc_chunk(cw_data_buf
, data_size
,
1714 cw_oob_buf
+ host
->bbm_size
,
1718 mtd
->ecc_stats
.failed
++;
1720 mtd
->ecc_stats
.corrected
+= ret
;
1721 max_bitflips
= max_t(unsigned int, max_bitflips
, ret
);
1725 return max_bitflips
;
1729 * reads back status registers set by the controller to notify page read
1730 * errors. this is equivalent to what 'ecc->correct()' would do.
1732 static int parse_read_errors(struct qcom_nand_host
*host
, u8
*data_buf
,
1733 u8
*oob_buf
, int page
)
1735 struct nand_chip
*chip
= &host
->chip
;
1736 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1737 struct mtd_info
*mtd
= nand_to_mtd(chip
);
1738 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1739 unsigned int max_bitflips
= 0, uncorrectable_cws
= 0;
1740 struct read_stats
*buf
;
1741 bool flash_op_err
= false, erased
;
1743 u8
*data_buf_start
= data_buf
, *oob_buf_start
= oob_buf
;
1745 buf
= (struct read_stats
*)nandc
->reg_read_buf
;
1746 nandc_read_buffer_sync(nandc
, true);
1748 for (i
= 0; i
< ecc
->steps
; i
++, buf
++) {
1749 u32 flash
, buffer
, erased_cw
;
1750 int data_len
, oob_len
;
1752 if (i
== (ecc
->steps
- 1)) {
1753 data_len
= ecc
->size
- ((ecc
->steps
- 1) << 2);
1754 oob_len
= ecc
->steps
<< 2;
1756 data_len
= host
->cw_data
;
1760 flash
= le32_to_cpu(buf
->flash
);
1761 buffer
= le32_to_cpu(buf
->buffer
);
1762 erased_cw
= le32_to_cpu(buf
->erased_cw
);
1765 * Check ECC failure for each codeword. ECC failure can
1766 * happen in either of the following conditions
1767 * 1. If number of bitflips are greater than ECC engine
1769 * 2. If this codeword contains all 0xff for which erased
1770 * codeword detection check will be done.
1772 if ((flash
& FS_OP_ERR
) && (buffer
& BS_UNCORRECTABLE_BIT
)) {
1774 * For BCH ECC, ignore erased codeword errors, if
1775 * ERASED_CW bits are set.
1777 if (host
->bch_enabled
) {
1778 erased
= (erased_cw
& ERASED_CW
) == ERASED_CW
?
1781 * For RS ECC, HW reports the erased CW by placing
1782 * special characters at certain offsets in the buffer.
1783 * These special characters will be valid only if
1784 * complete page is read i.e. data_buf is not NULL.
1786 } else if (data_buf
) {
1787 erased
= erased_chunk_check_and_fixup(data_buf
,
1794 uncorrectable_cws
|= BIT(i
);
1796 * Check if MPU or any other operational error (timeout,
1797 * device failure, etc.) happened for this codeword and
1798 * make flash_op_err true. If flash_op_err is set, then
1799 * EIO will be returned for page read.
1801 } else if (flash
& (FS_OP_ERR
| FS_MPU_ERR
)) {
1802 flash_op_err
= true;
1804 * No ECC or operational errors happened. Check the number of
1805 * bits corrected and update the ecc_stats.corrected.
1810 stat
= buffer
& BS_CORRECTABLE_ERR_MSK
;
1811 mtd
->ecc_stats
.corrected
+= stat
;
1812 max_bitflips
= max(max_bitflips
, stat
);
1816 data_buf
+= data_len
;
1818 oob_buf
+= oob_len
+ ecc
->bytes
;
1824 if (!uncorrectable_cws
)
1825 return max_bitflips
;
1827 return check_for_erased_page(host
, data_buf_start
, oob_buf_start
,
1828 uncorrectable_cws
, page
,
1833 * helper to perform the actual page read operation, used by ecc->read_page(),
1836 static int read_page_ecc(struct qcom_nand_host
*host
, u8
*data_buf
,
1837 u8
*oob_buf
, int page
)
1839 struct nand_chip
*chip
= &host
->chip
;
1840 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1841 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1842 u8
*data_buf_start
= data_buf
, *oob_buf_start
= oob_buf
;
1845 config_nand_page_read(nandc
);
1847 /* queue cmd descs for each codeword */
1848 for (i
= 0; i
< ecc
->steps
; i
++) {
1849 int data_size
, oob_size
;
1851 if (i
== (ecc
->steps
- 1)) {
1852 data_size
= ecc
->size
- ((ecc
->steps
- 1) << 2);
1853 oob_size
= (ecc
->steps
<< 2) + host
->ecc_bytes_hw
+
1856 data_size
= host
->cw_data
;
1857 oob_size
= host
->ecc_bytes_hw
+ host
->spare_bytes
;
1860 if (nandc
->props
->is_bam
) {
1861 if (data_buf
&& oob_buf
) {
1862 nandc_set_read_loc(nandc
, 0, 0, data_size
, 0);
1863 nandc_set_read_loc(nandc
, 1, data_size
,
1865 } else if (data_buf
) {
1866 nandc_set_read_loc(nandc
, 0, 0, data_size
, 1);
1868 nandc_set_read_loc(nandc
, 0, data_size
,
1873 config_nand_cw_read(nandc
, true);
1876 read_data_dma(nandc
, FLASH_BUF_ACC
, data_buf
,
1880 * when ecc is enabled, the controller doesn't read the real
1881 * or dummy bad block markers in each chunk. To maintain a
1882 * consistent layout across RAW and ECC reads, we just
1883 * leave the real/dummy BBM offsets empty (i.e, filled with
1889 for (j
= 0; j
< host
->bbm_size
; j
++)
1892 read_data_dma(nandc
, FLASH_BUF_ACC
+ data_size
,
1893 oob_buf
, oob_size
, 0);
1897 data_buf
+= data_size
;
1899 oob_buf
+= oob_size
;
1902 ret
= submit_descs(nandc
);
1906 dev_err(nandc
->dev
, "failure to read page/oob\n");
1910 return parse_read_errors(host
, data_buf_start
, oob_buf_start
, page
);
1914 * a helper that copies the last step/codeword of a page (containing free oob)
1915 * into our local buffer
1917 static int copy_last_cw(struct qcom_nand_host
*host
, int page
)
1919 struct nand_chip
*chip
= &host
->chip
;
1920 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1921 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1925 clear_read_regs(nandc
);
1927 size
= host
->use_ecc
? host
->cw_data
: host
->cw_size
;
1929 /* prepare a clean read buffer */
1930 memset(nandc
->data_buffer
, 0xff, size
);
1932 set_address(host
, host
->cw_size
* (ecc
->steps
- 1), page
);
1933 update_rw_regs(host
, 1, true);
1935 config_nand_single_cw_page_read(nandc
, host
->use_ecc
);
1937 read_data_dma(nandc
, FLASH_BUF_ACC
, nandc
->data_buffer
, size
, 0);
1939 ret
= submit_descs(nandc
);
1941 dev_err(nandc
->dev
, "failed to copy last codeword\n");
1948 /* implements ecc->read_page() */
1949 static int qcom_nandc_read_page(struct nand_chip
*chip
, uint8_t *buf
,
1950 int oob_required
, int page
)
1952 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
1953 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1954 u8
*data_buf
, *oob_buf
= NULL
;
1956 nand_read_page_op(chip
, page
, 0, NULL
, 0);
1958 oob_buf
= oob_required
? chip
->oob_poi
: NULL
;
1960 clear_bam_transaction(nandc
);
1962 return read_page_ecc(host
, data_buf
, oob_buf
, page
);
1965 /* implements ecc->read_page_raw() */
1966 static int qcom_nandc_read_page_raw(struct nand_chip
*chip
, uint8_t *buf
,
1967 int oob_required
, int page
)
1969 struct mtd_info
*mtd
= nand_to_mtd(chip
);
1970 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
1971 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1973 u8
*data_buf
= buf
, *oob_buf
= chip
->oob_poi
;
1975 for (cw
= 0; cw
< ecc
->steps
; cw
++) {
1976 ret
= qcom_nandc_read_cw_raw(mtd
, chip
, data_buf
, oob_buf
,
1981 data_buf
+= host
->cw_data
;
1982 oob_buf
+= ecc
->bytes
;
1988 /* implements ecc->read_oob() */
1989 static int qcom_nandc_read_oob(struct nand_chip
*chip
, int page
)
1991 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
1992 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
1993 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
1995 clear_read_regs(nandc
);
1996 clear_bam_transaction(nandc
);
1998 host
->use_ecc
= true;
1999 set_address(host
, 0, page
);
2000 update_rw_regs(host
, ecc
->steps
, true);
2002 return read_page_ecc(host
, NULL
, chip
->oob_poi
, page
);
2005 /* implements ecc->write_page() */
2006 static int qcom_nandc_write_page(struct nand_chip
*chip
, const uint8_t *buf
,
2007 int oob_required
, int page
)
2009 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2010 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2011 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2012 u8
*data_buf
, *oob_buf
;
2015 nand_prog_page_begin_op(chip
, page
, 0, NULL
, 0);
2017 clear_read_regs(nandc
);
2018 clear_bam_transaction(nandc
);
2020 data_buf
= (u8
*)buf
;
2021 oob_buf
= chip
->oob_poi
;
2023 host
->use_ecc
= true;
2024 update_rw_regs(host
, ecc
->steps
, false);
2025 config_nand_page_write(nandc
);
2027 for (i
= 0; i
< ecc
->steps
; i
++) {
2028 int data_size
, oob_size
;
2030 if (i
== (ecc
->steps
- 1)) {
2031 data_size
= ecc
->size
- ((ecc
->steps
- 1) << 2);
2032 oob_size
= (ecc
->steps
<< 2) + host
->ecc_bytes_hw
+
2035 data_size
= host
->cw_data
;
2036 oob_size
= ecc
->bytes
;
2040 write_data_dma(nandc
, FLASH_BUF_ACC
, data_buf
, data_size
,
2041 i
== (ecc
->steps
- 1) ? NAND_BAM_NO_EOT
: 0);
2044 * when ECC is enabled, we don't really need to write anything
2045 * to oob for the first n - 1 codewords since these oob regions
2046 * just contain ECC bytes that's written by the controller
2047 * itself. For the last codeword, we skip the bbm positions and
2048 * write to the free oob area.
2050 if (i
== (ecc
->steps
- 1)) {
2051 oob_buf
+= host
->bbm_size
;
2053 write_data_dma(nandc
, FLASH_BUF_ACC
+ data_size
,
2054 oob_buf
, oob_size
, 0);
2057 config_nand_cw_write(nandc
);
2059 data_buf
+= data_size
;
2060 oob_buf
+= oob_size
;
2063 ret
= submit_descs(nandc
);
2065 dev_err(nandc
->dev
, "failure to write page\n");
2070 ret
= nand_prog_page_end_op(chip
);
2075 /* implements ecc->write_page_raw() */
2076 static int qcom_nandc_write_page_raw(struct nand_chip
*chip
,
2077 const uint8_t *buf
, int oob_required
,
2080 struct mtd_info
*mtd
= nand_to_mtd(chip
);
2081 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2082 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2083 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2084 u8
*data_buf
, *oob_buf
;
2087 nand_prog_page_begin_op(chip
, page
, 0, NULL
, 0);
2088 clear_read_regs(nandc
);
2089 clear_bam_transaction(nandc
);
2091 data_buf
= (u8
*)buf
;
2092 oob_buf
= chip
->oob_poi
;
2094 host
->use_ecc
= false;
2095 update_rw_regs(host
, ecc
->steps
, false);
2096 config_nand_page_write(nandc
);
2098 for (i
= 0; i
< ecc
->steps
; i
++) {
2099 int data_size1
, data_size2
, oob_size1
, oob_size2
;
2100 int reg_off
= FLASH_BUF_ACC
;
2102 data_size1
= mtd
->writesize
- host
->cw_size
* (ecc
->steps
- 1);
2103 oob_size1
= host
->bbm_size
;
2105 if (i
== (ecc
->steps
- 1)) {
2106 data_size2
= ecc
->size
- data_size1
-
2107 ((ecc
->steps
- 1) << 2);
2108 oob_size2
= (ecc
->steps
<< 2) + host
->ecc_bytes_hw
+
2111 data_size2
= host
->cw_data
- data_size1
;
2112 oob_size2
= host
->ecc_bytes_hw
+ host
->spare_bytes
;
2115 write_data_dma(nandc
, reg_off
, data_buf
, data_size1
,
2117 reg_off
+= data_size1
;
2118 data_buf
+= data_size1
;
2120 write_data_dma(nandc
, reg_off
, oob_buf
, oob_size1
,
2122 reg_off
+= oob_size1
;
2123 oob_buf
+= oob_size1
;
2125 write_data_dma(nandc
, reg_off
, data_buf
, data_size2
,
2127 reg_off
+= data_size2
;
2128 data_buf
+= data_size2
;
2130 write_data_dma(nandc
, reg_off
, oob_buf
, oob_size2
, 0);
2131 oob_buf
+= oob_size2
;
2133 config_nand_cw_write(nandc
);
2136 ret
= submit_descs(nandc
);
2138 dev_err(nandc
->dev
, "failure to write raw page\n");
2143 ret
= nand_prog_page_end_op(chip
);
2149 * implements ecc->write_oob()
2151 * the NAND controller cannot write only data or only OOB within a codeword
2152 * since ECC is calculated for the combined codeword. So update the OOB from
2153 * chip->oob_poi, and pad the data area with OxFF before writing.
2155 static int qcom_nandc_write_oob(struct nand_chip
*chip
, int page
)
2157 struct mtd_info
*mtd
= nand_to_mtd(chip
);
2158 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2159 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2160 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2161 u8
*oob
= chip
->oob_poi
;
2162 int data_size
, oob_size
;
2165 host
->use_ecc
= true;
2166 clear_bam_transaction(nandc
);
2168 /* calculate the data and oob size for the last codeword/step */
2169 data_size
= ecc
->size
- ((ecc
->steps
- 1) << 2);
2170 oob_size
= mtd
->oobavail
;
2172 memset(nandc
->data_buffer
, 0xff, host
->cw_data
);
2173 /* override new oob content to last codeword */
2174 mtd_ooblayout_get_databytes(mtd
, nandc
->data_buffer
+ data_size
, oob
,
2177 set_address(host
, host
->cw_size
* (ecc
->steps
- 1), page
);
2178 update_rw_regs(host
, 1, false);
2180 config_nand_page_write(nandc
);
2181 write_data_dma(nandc
, FLASH_BUF_ACC
,
2182 nandc
->data_buffer
, data_size
+ oob_size
, 0);
2183 config_nand_cw_write(nandc
);
2185 ret
= submit_descs(nandc
);
2190 dev_err(nandc
->dev
, "failure to write oob\n");
2194 return nand_prog_page_end_op(chip
);
2197 static int qcom_nandc_block_bad(struct nand_chip
*chip
, loff_t ofs
)
2199 struct mtd_info
*mtd
= nand_to_mtd(chip
);
2200 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2201 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2202 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2203 int page
, ret
, bbpos
, bad
= 0;
2205 page
= (int)(ofs
>> chip
->page_shift
) & chip
->pagemask
;
2208 * configure registers for a raw sub page read, the address is set to
2209 * the beginning of the last codeword, we don't care about reading ecc
2210 * portion of oob. we just want the first few bytes from this codeword
2211 * that contains the BBM
2213 host
->use_ecc
= false;
2215 clear_bam_transaction(nandc
);
2216 ret
= copy_last_cw(host
, page
);
2220 if (check_flash_errors(host
, 1)) {
2221 dev_warn(nandc
->dev
, "error when trying to read BBM\n");
2225 bbpos
= mtd
->writesize
- host
->cw_size
* (ecc
->steps
- 1);
2227 bad
= nandc
->data_buffer
[bbpos
] != 0xff;
2229 if (chip
->options
& NAND_BUSWIDTH_16
)
2230 bad
= bad
|| (nandc
->data_buffer
[bbpos
+ 1] != 0xff);
2235 static int qcom_nandc_block_markbad(struct nand_chip
*chip
, loff_t ofs
)
2237 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2238 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2239 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2242 clear_read_regs(nandc
);
2243 clear_bam_transaction(nandc
);
2246 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2247 * we don't care about the rest of the content in the codeword since
2248 * we aren't going to use this block again
2250 memset(nandc
->data_buffer
, 0x00, host
->cw_size
);
2252 page
= (int)(ofs
>> chip
->page_shift
) & chip
->pagemask
;
2255 host
->use_ecc
= false;
2256 set_address(host
, host
->cw_size
* (ecc
->steps
- 1), page
);
2257 update_rw_regs(host
, 1, false);
2259 config_nand_page_write(nandc
);
2260 write_data_dma(nandc
, FLASH_BUF_ACC
,
2261 nandc
->data_buffer
, host
->cw_size
, 0);
2262 config_nand_cw_write(nandc
);
2264 ret
= submit_descs(nandc
);
2269 dev_err(nandc
->dev
, "failure to update BBM\n");
2273 return nand_prog_page_end_op(chip
);
2277 * the three functions below implement chip->legacy.read_byte(),
2278 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these
2279 * aren't used for reading/writing page data, they are used for smaller data
2280 * like reading id, status etc
2282 static uint8_t qcom_nandc_read_byte(struct nand_chip
*chip
)
2284 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2285 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2286 u8
*buf
= nandc
->data_buffer
;
2289 if (host
->last_command
== NAND_CMD_STATUS
) {
2292 host
->status
= NAND_STATUS_READY
| NAND_STATUS_WP
;
2297 if (nandc
->buf_start
< nandc
->buf_count
)
2298 ret
= buf
[nandc
->buf_start
++];
2303 static void qcom_nandc_read_buf(struct nand_chip
*chip
, uint8_t *buf
, int len
)
2305 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2306 int real_len
= min_t(size_t, len
, nandc
->buf_count
- nandc
->buf_start
);
2308 memcpy(buf
, nandc
->data_buffer
+ nandc
->buf_start
, real_len
);
2309 nandc
->buf_start
+= real_len
;
2312 static void qcom_nandc_write_buf(struct nand_chip
*chip
, const uint8_t *buf
,
2315 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2316 int real_len
= min_t(size_t, len
, nandc
->buf_count
- nandc
->buf_start
);
2318 memcpy(nandc
->data_buffer
+ nandc
->buf_start
, buf
, real_len
);
2320 nandc
->buf_start
+= real_len
;
2323 /* we support only one external chip for now */
2324 static void qcom_nandc_select_chip(struct nand_chip
*chip
, int chipnr
)
2326 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2331 dev_warn(nandc
->dev
, "invalid chip select\n");
2335 * NAND controller page layout info
2337 * Layout with ECC enabled:
2339 * |----------------------| |---------------------------------|
2340 * | xx.......yy| | *********xx.......yy|
2341 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
2342 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
2343 * | xx.......yy| | *********xx.......yy|
2344 * |----------------------| |---------------------------------|
2345 * codeword 1,2..n-1 codeword n
2346 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
2348 * n = Number of codewords in the page
2350 * * = Spare/free bytes
2351 * x = Unused byte(s)
2352 * y = Reserved byte(s)
2354 * 2K page: n = 4, spare = 16 bytes
2355 * 4K page: n = 8, spare = 32 bytes
2356 * 8K page: n = 16, spare = 64 bytes
2358 * the qcom nand controller operates at a sub page/codeword level. each
2359 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2360 * the number of ECC bytes vary based on the ECC strength and the bus width.
2362 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2363 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2364 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2366 * When we access a page with ECC enabled, the reserved bytes(s) are not
2367 * accessible at all. When reading, we fill up these unreadable positions
2368 * with 0xffs. When writing, the controller skips writing the inaccessible
2371 * Layout with ECC disabled:
2373 * |------------------------------| |---------------------------------------|
2374 * | yy xx.......| | bb *********xx.......|
2375 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
2376 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
2377 * | yy xx.......| | bb *********xx.......|
2378 * |------------------------------| |---------------------------------------|
2379 * codeword 1,2..n-1 codeword n
2380 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
2382 * n = Number of codewords in the page
2384 * * = Spare/free bytes
2385 * x = Unused byte(s)
2386 * y = Dummy Bad Bock byte(s)
2387 * b = Real Bad Block byte(s)
2388 * size1/size2 = function of codeword size and 'n'
2390 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2391 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2392 * Block Markers. In the last codeword, this position contains the real BBM
2394 * In order to have a consistent layout between RAW and ECC modes, we assume
2395 * the following OOB layout arrangement:
2397 * |-----------| |--------------------|
2398 * |yyxx.......| |bb*********xx.......|
2399 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
2400 * |yyxx.......| |bb*********xx.......|
2401 * |yyxx.......| |bb*********xx.......|
2402 * |-----------| |--------------------|
2403 * first n - 1 nth OOB region
2406 * n = Number of codewords in the page
2408 * * = FREE OOB bytes
2409 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2410 * x = Unused byte(s)
2411 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2413 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2414 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2415 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2416 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2417 * the sum of the three).
2419 static int qcom_nand_ooblayout_ecc(struct mtd_info
*mtd
, int section
,
2420 struct mtd_oob_region
*oobregion
)
2422 struct nand_chip
*chip
= mtd_to_nand(mtd
);
2423 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2424 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2430 oobregion
->length
= (ecc
->bytes
* (ecc
->steps
- 1)) +
2432 oobregion
->offset
= 0;
2434 oobregion
->length
= host
->ecc_bytes_hw
+ host
->spare_bytes
;
2435 oobregion
->offset
= mtd
->oobsize
- oobregion
->length
;
2441 static int qcom_nand_ooblayout_free(struct mtd_info
*mtd
, int section
,
2442 struct mtd_oob_region
*oobregion
)
2444 struct nand_chip
*chip
= mtd_to_nand(mtd
);
2445 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2446 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2451 oobregion
->length
= ecc
->steps
* 4;
2452 oobregion
->offset
= ((ecc
->steps
- 1) * ecc
->bytes
) + host
->bbm_size
;
2457 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops
= {
2458 .ecc
= qcom_nand_ooblayout_ecc
,
2459 .free
= qcom_nand_ooblayout_free
,
2463 qcom_nandc_calc_ecc_bytes(int step_size
, int strength
)
2465 return strength
== 4 ? 12 : 16;
2467 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps
, qcom_nandc_calc_ecc_bytes
,
2468 NANDC_STEP_SIZE
, 4, 8);
2470 static int qcom_nand_attach_chip(struct nand_chip
*chip
)
2472 struct mtd_info
*mtd
= nand_to_mtd(chip
);
2473 struct qcom_nand_host
*host
= to_qcom_nand_host(chip
);
2474 struct nand_ecc_ctrl
*ecc
= &chip
->ecc
;
2475 struct qcom_nand_controller
*nandc
= get_qcom_nand_controller(chip
);
2476 int cwperpage
, bad_block_byte
, ret
;
2480 /* controller only supports 512 bytes data steps */
2481 ecc
->size
= NANDC_STEP_SIZE
;
2482 wide_bus
= chip
->options
& NAND_BUSWIDTH_16
? true : false;
2483 cwperpage
= mtd
->writesize
/ NANDC_STEP_SIZE
;
2486 * Each CW has 4 available OOB bytes which will be protected with ECC
2487 * so remaining bytes can be used for ECC.
2489 ret
= nand_ecc_choose_conf(chip
, &qcom_nandc_ecc_caps
,
2490 mtd
->oobsize
- (cwperpage
* 4));
2492 dev_err(nandc
->dev
, "No valid ECC settings possible\n");
2496 if (ecc
->strength
>= 8) {
2497 /* 8 bit ECC defaults to BCH ECC on all platforms */
2498 host
->bch_enabled
= true;
2502 host
->ecc_bytes_hw
= 14;
2503 host
->spare_bytes
= 0;
2506 host
->ecc_bytes_hw
= 13;
2507 host
->spare_bytes
= 2;
2512 * if the controller supports BCH for 4 bit ECC, the controller
2513 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2516 if (nandc
->props
->ecc_modes
& ECC_BCH_4BIT
) {
2518 host
->bch_enabled
= true;
2522 host
->ecc_bytes_hw
= 8;
2523 host
->spare_bytes
= 2;
2526 host
->ecc_bytes_hw
= 7;
2527 host
->spare_bytes
= 4;
2532 host
->ecc_bytes_hw
= 10;
2535 host
->spare_bytes
= 0;
2538 host
->spare_bytes
= 1;
2545 * we consider ecc->bytes as the sum of all the non-data content in a
2546 * step. It gives us a clean representation of the oob area (even if
2547 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2548 * ECC and 12 bytes for 4 bit ECC
2550 ecc
->bytes
= host
->ecc_bytes_hw
+ host
->spare_bytes
+ host
->bbm_size
;
2552 ecc
->read_page
= qcom_nandc_read_page
;
2553 ecc
->read_page_raw
= qcom_nandc_read_page_raw
;
2554 ecc
->read_oob
= qcom_nandc_read_oob
;
2555 ecc
->write_page
= qcom_nandc_write_page
;
2556 ecc
->write_page_raw
= qcom_nandc_write_page_raw
;
2557 ecc
->write_oob
= qcom_nandc_write_oob
;
2559 ecc
->mode
= NAND_ECC_HW
;
2561 mtd_set_ooblayout(mtd
, &qcom_nand_ooblayout_ops
);
2563 nandc
->max_cwperpage
= max_t(unsigned int, nandc
->max_cwperpage
,
2567 * DATA_UD_BYTES varies based on whether the read/write command protects
2568 * spare data with ECC too. We protect spare data by default, so we set
2569 * it to main + spare data, which are 512 and 4 bytes respectively.
2571 host
->cw_data
= 516;
2574 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2577 host
->cw_size
= host
->cw_data
+ ecc
->bytes
;
2578 bad_block_byte
= mtd
->writesize
- host
->cw_size
* (cwperpage
- 1) + 1;
2580 host
->cfg0
= (cwperpage
- 1) << CW_PER_PAGE
2581 | host
->cw_data
<< UD_SIZE_BYTES
2582 | 0 << DISABLE_STATUS_AFTER_WRITE
2583 | 5 << NUM_ADDR_CYCLES
2584 | host
->ecc_bytes_hw
<< ECC_PARITY_SIZE_BYTES_RS
2585 | 0 << STATUS_BFR_READ
2586 | 1 << SET_RD_MODE_AFTER_STATUS
2587 | host
->spare_bytes
<< SPARE_SIZE_BYTES
;
2589 host
->cfg1
= 7 << NAND_RECOVERY_CYCLES
2590 | 0 << CS_ACTIVE_BSY
2591 | bad_block_byte
<< BAD_BLOCK_BYTE_NUM
2592 | 0 << BAD_BLOCK_IN_SPARE_AREA
2593 | 2 << WR_RD_BSY_GAP
2594 | wide_bus
<< WIDE_FLASH
2595 | host
->bch_enabled
<< ENABLE_BCH_ECC
;
2597 host
->cfg0_raw
= (cwperpage
- 1) << CW_PER_PAGE
2598 | host
->cw_size
<< UD_SIZE_BYTES
2599 | 5 << NUM_ADDR_CYCLES
2600 | 0 << SPARE_SIZE_BYTES
;
2602 host
->cfg1_raw
= 7 << NAND_RECOVERY_CYCLES
2603 | 0 << CS_ACTIVE_BSY
2604 | 17 << BAD_BLOCK_BYTE_NUM
2605 | 1 << BAD_BLOCK_IN_SPARE_AREA
2606 | 2 << WR_RD_BSY_GAP
2607 | wide_bus
<< WIDE_FLASH
2608 | 1 << DEV0_CFG1_ECC_DISABLE
;
2610 host
->ecc_bch_cfg
= !host
->bch_enabled
<< ECC_CFG_ECC_DISABLE
2612 | host
->cw_data
<< ECC_NUM_DATA_BYTES
2613 | 1 << ECC_FORCE_CLK_OPEN
2614 | ecc_mode
<< ECC_MODE
2615 | host
->ecc_bytes_hw
<< ECC_PARITY_SIZE_BYTES_BCH
;
2617 host
->ecc_buf_cfg
= 0x203 << NUM_STEPS
;
2619 host
->clrflashstatus
= FS_READY_BSY_N
;
2620 host
->clrreadstatus
= 0xc0;
2621 nandc
->regs
->erased_cw_detect_cfg_clr
=
2622 cpu_to_le32(CLR_ERASED_PAGE_DET
);
2623 nandc
->regs
->erased_cw_detect_cfg_set
=
2624 cpu_to_le32(SET_ERASED_PAGE_DET
);
2627 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2628 host
->cfg0
, host
->cfg1
, host
->ecc_buf_cfg
, host
->ecc_bch_cfg
,
2629 host
->cw_size
, host
->cw_data
, ecc
->strength
, ecc
->bytes
,
2635 static const struct nand_controller_ops qcom_nandc_ops
= {
2636 .attach_chip
= qcom_nand_attach_chip
,
2639 static int qcom_nandc_alloc(struct qcom_nand_controller
*nandc
)
2643 ret
= dma_set_coherent_mask(nandc
->dev
, DMA_BIT_MASK(32));
2645 dev_err(nandc
->dev
, "failed to set DMA mask\n");
2650 * we use the internal buffer for reading ONFI params, reading small
2651 * data like ID and status, and preforming read-copy-write operations
2652 * when writing to a codeword partially. 532 is the maximum possible
2653 * size of a codeword for our nand controller
2655 nandc
->buf_size
= 532;
2657 nandc
->data_buffer
= devm_kzalloc(nandc
->dev
, nandc
->buf_size
,
2659 if (!nandc
->data_buffer
)
2662 nandc
->regs
= devm_kzalloc(nandc
->dev
, sizeof(*nandc
->regs
),
2667 nandc
->reg_read_buf
= devm_kcalloc(nandc
->dev
,
2668 MAX_REG_RD
, sizeof(*nandc
->reg_read_buf
),
2670 if (!nandc
->reg_read_buf
)
2673 if (nandc
->props
->is_bam
) {
2674 nandc
->reg_read_dma
=
2675 dma_map_single(nandc
->dev
, nandc
->reg_read_buf
,
2677 sizeof(*nandc
->reg_read_buf
),
2679 if (dma_mapping_error(nandc
->dev
, nandc
->reg_read_dma
)) {
2680 dev_err(nandc
->dev
, "failed to DMA MAP reg buffer\n");
2684 nandc
->tx_chan
= dma_request_slave_channel(nandc
->dev
, "tx");
2685 if (!nandc
->tx_chan
) {
2686 dev_err(nandc
->dev
, "failed to request tx channel\n");
2690 nandc
->rx_chan
= dma_request_slave_channel(nandc
->dev
, "rx");
2691 if (!nandc
->rx_chan
) {
2692 dev_err(nandc
->dev
, "failed to request rx channel\n");
2696 nandc
->cmd_chan
= dma_request_slave_channel(nandc
->dev
, "cmd");
2697 if (!nandc
->cmd_chan
) {
2698 dev_err(nandc
->dev
, "failed to request cmd channel\n");
2703 * Initially allocate BAM transaction to read ONFI param page.
2704 * After detecting all the devices, this BAM transaction will
2705 * be freed and the next BAM tranasction will be allocated with
2706 * maximum codeword size
2708 nandc
->max_cwperpage
= 1;
2709 nandc
->bam_txn
= alloc_bam_transaction(nandc
);
2710 if (!nandc
->bam_txn
) {
2712 "failed to allocate bam transaction\n");
2716 nandc
->chan
= dma_request_slave_channel(nandc
->dev
, "rxtx");
2719 "failed to request slave channel\n");
2724 INIT_LIST_HEAD(&nandc
->desc_list
);
2725 INIT_LIST_HEAD(&nandc
->host_list
);
2727 nand_controller_init(&nandc
->controller
);
2728 nandc
->controller
.ops
= &qcom_nandc_ops
;
2733 static void qcom_nandc_unalloc(struct qcom_nand_controller
*nandc
)
2735 if (nandc
->props
->is_bam
) {
2736 if (!dma_mapping_error(nandc
->dev
, nandc
->reg_read_dma
))
2737 dma_unmap_single(nandc
->dev
, nandc
->reg_read_dma
,
2739 sizeof(*nandc
->reg_read_buf
),
2743 dma_release_channel(nandc
->tx_chan
);
2746 dma_release_channel(nandc
->rx_chan
);
2748 if (nandc
->cmd_chan
)
2749 dma_release_channel(nandc
->cmd_chan
);
2752 dma_release_channel(nandc
->chan
);
2756 /* one time setup of a few nand controller registers */
2757 static int qcom_nandc_setup(struct qcom_nand_controller
*nandc
)
2762 nandc_write(nandc
, SFLASHC_BURST_CFG
, 0);
2763 nandc_write(nandc
, dev_cmd_reg_addr(nandc
, NAND_DEV_CMD_VLD
),
2764 NAND_DEV_CMD_VLD_VAL
);
2766 /* enable ADM or BAM DMA */
2767 if (nandc
->props
->is_bam
) {
2768 nand_ctrl
= nandc_read(nandc
, NAND_CTRL
);
2769 nandc_write(nandc
, NAND_CTRL
, nand_ctrl
| BAM_MODE_EN
);
2771 nandc_write(nandc
, NAND_FLASH_CHIP_SELECT
, DM_EN
);
2774 /* save the original values of these registers */
2775 nandc
->cmd1
= nandc_read(nandc
, dev_cmd_reg_addr(nandc
, NAND_DEV_CMD1
));
2776 nandc
->vld
= NAND_DEV_CMD_VLD_VAL
;
2781 static int qcom_nand_host_init_and_register(struct qcom_nand_controller
*nandc
,
2782 struct qcom_nand_host
*host
,
2783 struct device_node
*dn
)
2785 struct nand_chip
*chip
= &host
->chip
;
2786 struct mtd_info
*mtd
= nand_to_mtd(chip
);
2787 struct device
*dev
= nandc
->dev
;
2790 ret
= of_property_read_u32(dn
, "reg", &host
->cs
);
2792 dev_err(dev
, "can't get chip-select\n");
2796 nand_set_flash_node(chip
, dn
);
2797 mtd
->name
= devm_kasprintf(dev
, GFP_KERNEL
, "qcom_nand.%d", host
->cs
);
2801 mtd
->owner
= THIS_MODULE
;
2802 mtd
->dev
.parent
= dev
;
2804 chip
->legacy
.cmdfunc
= qcom_nandc_command
;
2805 chip
->legacy
.select_chip
= qcom_nandc_select_chip
;
2806 chip
->legacy
.read_byte
= qcom_nandc_read_byte
;
2807 chip
->legacy
.read_buf
= qcom_nandc_read_buf
;
2808 chip
->legacy
.write_buf
= qcom_nandc_write_buf
;
2809 chip
->legacy
.set_features
= nand_get_set_features_notsupp
;
2810 chip
->legacy
.get_features
= nand_get_set_features_notsupp
;
2813 * the bad block marker is readable only when we read the last codeword
2814 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2815 * helpers don't allow us to read BB from a nand chip with ECC
2816 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2817 * and block_markbad helpers until we permanently switch to using
2818 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2820 chip
->legacy
.block_bad
= qcom_nandc_block_bad
;
2821 chip
->legacy
.block_markbad
= qcom_nandc_block_markbad
;
2823 chip
->controller
= &nandc
->controller
;
2824 chip
->options
|= NAND_NO_SUBPAGE_WRITE
| NAND_USE_BOUNCE_BUFFER
|
2827 /* set up initial status value */
2828 host
->status
= NAND_STATUS_READY
| NAND_STATUS_WP
;
2830 ret
= nand_scan(chip
, 1);
2834 if (nandc
->props
->is_bam
) {
2835 free_bam_transaction(nandc
);
2836 nandc
->bam_txn
= alloc_bam_transaction(nandc
);
2837 if (!nandc
->bam_txn
) {
2839 "failed to allocate bam transaction\n");
2844 ret
= mtd_device_register(mtd
, NULL
, 0);
2851 static int qcom_probe_nand_devices(struct qcom_nand_controller
*nandc
)
2853 struct device
*dev
= nandc
->dev
;
2854 struct device_node
*dn
= dev
->of_node
, *child
;
2855 struct qcom_nand_host
*host
;
2858 for_each_available_child_of_node(dn
, child
) {
2859 host
= devm_kzalloc(dev
, sizeof(*host
), GFP_KERNEL
);
2865 ret
= qcom_nand_host_init_and_register(nandc
, host
, child
);
2867 devm_kfree(dev
, host
);
2871 list_add_tail(&host
->node
, &nandc
->host_list
);
2874 if (list_empty(&nandc
->host_list
))
2880 /* parse custom DT properties here */
2881 static int qcom_nandc_parse_dt(struct platform_device
*pdev
)
2883 struct qcom_nand_controller
*nandc
= platform_get_drvdata(pdev
);
2884 struct device_node
*np
= nandc
->dev
->of_node
;
2887 if (!nandc
->props
->is_bam
) {
2888 ret
= of_property_read_u32(np
, "qcom,cmd-crci",
2891 dev_err(nandc
->dev
, "command CRCI unspecified\n");
2895 ret
= of_property_read_u32(np
, "qcom,data-crci",
2898 dev_err(nandc
->dev
, "data CRCI unspecified\n");
2906 static int qcom_nandc_probe(struct platform_device
*pdev
)
2908 struct qcom_nand_controller
*nandc
;
2909 const void *dev_data
;
2910 struct device
*dev
= &pdev
->dev
;
2911 struct resource
*res
;
2914 nandc
= devm_kzalloc(&pdev
->dev
, sizeof(*nandc
), GFP_KERNEL
);
2918 platform_set_drvdata(pdev
, nandc
);
2921 dev_data
= of_device_get_match_data(dev
);
2923 dev_err(&pdev
->dev
, "failed to get device data\n");
2927 nandc
->props
= dev_data
;
2929 nandc
->core_clk
= devm_clk_get(dev
, "core");
2930 if (IS_ERR(nandc
->core_clk
))
2931 return PTR_ERR(nandc
->core_clk
);
2933 nandc
->aon_clk
= devm_clk_get(dev
, "aon");
2934 if (IS_ERR(nandc
->aon_clk
))
2935 return PTR_ERR(nandc
->aon_clk
);
2937 ret
= qcom_nandc_parse_dt(pdev
);
2941 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
2942 nandc
->base
= devm_ioremap_resource(dev
, res
);
2943 if (IS_ERR(nandc
->base
))
2944 return PTR_ERR(nandc
->base
);
2946 nandc
->base_phys
= res
->start
;
2947 nandc
->base_dma
= dma_map_resource(dev
, res
->start
,
2949 DMA_BIDIRECTIONAL
, 0);
2950 if (!nandc
->base_dma
)
2953 ret
= qcom_nandc_alloc(nandc
);
2955 goto err_nandc_alloc
;
2957 ret
= clk_prepare_enable(nandc
->core_clk
);
2961 ret
= clk_prepare_enable(nandc
->aon_clk
);
2965 ret
= qcom_nandc_setup(nandc
);
2969 ret
= qcom_probe_nand_devices(nandc
);
2976 clk_disable_unprepare(nandc
->aon_clk
);
2978 clk_disable_unprepare(nandc
->core_clk
);
2980 qcom_nandc_unalloc(nandc
);
2982 dma_unmap_resource(dev
, res
->start
, resource_size(res
),
2983 DMA_BIDIRECTIONAL
, 0);
2988 static int qcom_nandc_remove(struct platform_device
*pdev
)
2990 struct qcom_nand_controller
*nandc
= platform_get_drvdata(pdev
);
2991 struct resource
*res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
2992 struct qcom_nand_host
*host
;
2994 list_for_each_entry(host
, &nandc
->host_list
, node
)
2995 nand_release(&host
->chip
);
2998 qcom_nandc_unalloc(nandc
);
3000 clk_disable_unprepare(nandc
->aon_clk
);
3001 clk_disable_unprepare(nandc
->core_clk
);
3003 dma_unmap_resource(&pdev
->dev
, nandc
->base_dma
, resource_size(res
),
3004 DMA_BIDIRECTIONAL
, 0);
3009 static const struct qcom_nandc_props ipq806x_nandc_props
= {
3010 .ecc_modes
= (ECC_RS_4BIT
| ECC_BCH_8BIT
),
3012 .dev_cmd_reg_start
= 0x0,
3015 static const struct qcom_nandc_props ipq4019_nandc_props
= {
3016 .ecc_modes
= (ECC_BCH_4BIT
| ECC_BCH_8BIT
),
3018 .dev_cmd_reg_start
= 0x0,
3021 static const struct qcom_nandc_props ipq8074_nandc_props
= {
3022 .ecc_modes
= (ECC_BCH_4BIT
| ECC_BCH_8BIT
),
3024 .dev_cmd_reg_start
= 0x7000,
3028 * data will hold a struct pointer containing more differences once we support
3029 * more controller variants
3031 static const struct of_device_id qcom_nandc_of_match
[] = {
3033 .compatible
= "qcom,ipq806x-nand",
3034 .data
= &ipq806x_nandc_props
,
3037 .compatible
= "qcom,ipq4019-nand",
3038 .data
= &ipq4019_nandc_props
,
3041 .compatible
= "qcom,ipq8074-nand",
3042 .data
= &ipq8074_nandc_props
,
3046 MODULE_DEVICE_TABLE(of
, qcom_nandc_of_match
);
3048 static struct platform_driver qcom_nandc_driver
= {
3050 .name
= "qcom-nandc",
3051 .of_match_table
= qcom_nandc_of_match
,
3053 .probe
= qcom_nandc_probe
,
3054 .remove
= qcom_nandc_remove
,
3056 module_platform_driver(qcom_nandc_driver
);
3058 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3059 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3060 MODULE_LICENSE("GPL v2");