]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/mmc/host/dw_mmc.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
mmc: dw_mmc: Avoid accessing registers in runtime suspended state
[mirror_ubuntu-bionic-kernel.git] / drivers / mmc / host / dw_mmc.c
1 /*
2 * Synopsys DesignWare Multimedia Card Interface driver
3 * (Based on NXP driver for lpc 31xx)
4 *
5 * Copyright (C) 2009 NXP Semiconductors
6 * Copyright (C) 2009, 2010 Imagination Technologies Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 */
13
14 #include <linux/blkdev.h>
15 #include <linux/clk.h>
16 #include <linux/debugfs.h>
17 #include <linux/device.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/init.h>
21 #include <linux/interrupt.h>
22 #include <linux/iopoll.h>
23 #include <linux/ioport.h>
24 #include <linux/module.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/seq_file.h>
28 #include <linux/slab.h>
29 #include <linux/stat.h>
30 #include <linux/delay.h>
31 #include <linux/irq.h>
32 #include <linux/mmc/card.h>
33 #include <linux/mmc/host.h>
34 #include <linux/mmc/mmc.h>
35 #include <linux/mmc/sd.h>
36 #include <linux/mmc/sdio.h>
37 #include <linux/bitops.h>
38 #include <linux/regulator/consumer.h>
39 #include <linux/of.h>
40 #include <linux/of_gpio.h>
41 #include <linux/mmc/slot-gpio.h>
42
43 #include "dw_mmc.h"
44
45 /* Common flag combinations */
46 #define DW_MCI_DATA_ERROR_FLAGS (SDMMC_INT_DRTO | SDMMC_INT_DCRC | \
47 SDMMC_INT_HTO | SDMMC_INT_SBE | \
48 SDMMC_INT_EBE | SDMMC_INT_HLE)
49 #define DW_MCI_CMD_ERROR_FLAGS (SDMMC_INT_RTO | SDMMC_INT_RCRC | \
50 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE)
51 #define DW_MCI_ERROR_FLAGS (DW_MCI_DATA_ERROR_FLAGS | \
52 DW_MCI_CMD_ERROR_FLAGS)
53 #define DW_MCI_SEND_STATUS 1
54 #define DW_MCI_RECV_STATUS 2
55 #define DW_MCI_DMA_THRESHOLD 16
56
57 #define DW_MCI_FREQ_MAX 200000000 /* unit: HZ */
58 #define DW_MCI_FREQ_MIN 100000 /* unit: HZ */
59
60 #define IDMAC_INT_CLR (SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \
61 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \
62 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \
63 SDMMC_IDMAC_INT_TI)
64
65 #define DESC_RING_BUF_SZ PAGE_SIZE
66
67 struct idmac_desc_64addr {
68 u32 des0; /* Control Descriptor */
69 #define IDMAC_OWN_CLR64(x) \
70 !((x) & cpu_to_le32(IDMAC_DES0_OWN))
71
72 u32 des1; /* Reserved */
73
74 u32 des2; /*Buffer sizes */
75 #define IDMAC_64ADDR_SET_BUFFER1_SIZE(d, s) \
76 ((d)->des2 = ((d)->des2 & cpu_to_le32(0x03ffe000)) | \
77 ((cpu_to_le32(s)) & cpu_to_le32(0x1fff)))
78
79 u32 des3; /* Reserved */
80
81 u32 des4; /* Lower 32-bits of Buffer Address Pointer 1*/
82 u32 des5; /* Upper 32-bits of Buffer Address Pointer 1*/
83
84 u32 des6; /* Lower 32-bits of Next Descriptor Address */
85 u32 des7; /* Upper 32-bits of Next Descriptor Address */
86 };
87
88 struct idmac_desc {
89 __le32 des0; /* Control Descriptor */
90 #define IDMAC_DES0_DIC BIT(1)
91 #define IDMAC_DES0_LD BIT(2)
92 #define IDMAC_DES0_FD BIT(3)
93 #define IDMAC_DES0_CH BIT(4)
94 #define IDMAC_DES0_ER BIT(5)
95 #define IDMAC_DES0_CES BIT(30)
96 #define IDMAC_DES0_OWN BIT(31)
97
98 __le32 des1; /* Buffer sizes */
99 #define IDMAC_SET_BUFFER1_SIZE(d, s) \
100 ((d)->des1 = ((d)->des1 & cpu_to_le32(0x03ffe000)) | (cpu_to_le32((s) & 0x1fff)))
101
102 __le32 des2; /* buffer 1 physical address */
103
104 __le32 des3; /* buffer 2 physical address */
105 };
106
107 /* Each descriptor can transfer up to 4KB of data in chained mode */
108 #define DW_MCI_DESC_DATA_LENGTH 0x1000
109
110 #if defined(CONFIG_DEBUG_FS)
111 static int dw_mci_req_show(struct seq_file *s, void *v)
112 {
113 struct dw_mci_slot *slot = s->private;
114 struct mmc_request *mrq;
115 struct mmc_command *cmd;
116 struct mmc_command *stop;
117 struct mmc_data *data;
118
119 /* Make sure we get a consistent snapshot */
120 spin_lock_bh(&slot->host->lock);
121 mrq = slot->mrq;
122
123 if (mrq) {
124 cmd = mrq->cmd;
125 data = mrq->data;
126 stop = mrq->stop;
127
128 if (cmd)
129 seq_printf(s,
130 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
131 cmd->opcode, cmd->arg, cmd->flags,
132 cmd->resp[0], cmd->resp[1], cmd->resp[2],
133 cmd->resp[2], cmd->error);
134 if (data)
135 seq_printf(s, "DATA %u / %u * %u flg %x err %d\n",
136 data->bytes_xfered, data->blocks,
137 data->blksz, data->flags, data->error);
138 if (stop)
139 seq_printf(s,
140 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
141 stop->opcode, stop->arg, stop->flags,
142 stop->resp[0], stop->resp[1], stop->resp[2],
143 stop->resp[2], stop->error);
144 }
145
146 spin_unlock_bh(&slot->host->lock);
147
148 return 0;
149 }
150
151 static int dw_mci_req_open(struct inode *inode, struct file *file)
152 {
153 return single_open(file, dw_mci_req_show, inode->i_private);
154 }
155
156 static const struct file_operations dw_mci_req_fops = {
157 .owner = THIS_MODULE,
158 .open = dw_mci_req_open,
159 .read = seq_read,
160 .llseek = seq_lseek,
161 .release = single_release,
162 };
163
164 static int dw_mci_regs_show(struct seq_file *s, void *v)
165 {
166 struct dw_mci *host = s->private;
167
168 pm_runtime_get_sync(host->dev);
169
170 seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS));
171 seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS));
172 seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD));
173 seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL));
174 seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK));
175 seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA));
176
177 pm_runtime_put_autosuspend(host->dev);
178
179 return 0;
180 }
181
182 static int dw_mci_regs_open(struct inode *inode, struct file *file)
183 {
184 return single_open(file, dw_mci_regs_show, inode->i_private);
185 }
186
187 static const struct file_operations dw_mci_regs_fops = {
188 .owner = THIS_MODULE,
189 .open = dw_mci_regs_open,
190 .read = seq_read,
191 .llseek = seq_lseek,
192 .release = single_release,
193 };
194
195 static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
196 {
197 struct mmc_host *mmc = slot->mmc;
198 struct dw_mci *host = slot->host;
199 struct dentry *root;
200 struct dentry *node;
201
202 root = mmc->debugfs_root;
203 if (!root)
204 return;
205
206 node = debugfs_create_file("regs", S_IRUSR, root, host,
207 &dw_mci_regs_fops);
208 if (!node)
209 goto err;
210
211 node = debugfs_create_file("req", S_IRUSR, root, slot,
212 &dw_mci_req_fops);
213 if (!node)
214 goto err;
215
216 node = debugfs_create_u32("state", S_IRUSR, root, (u32 *)&host->state);
217 if (!node)
218 goto err;
219
220 node = debugfs_create_x32("pending_events", S_IRUSR, root,
221 (u32 *)&host->pending_events);
222 if (!node)
223 goto err;
224
225 node = debugfs_create_x32("completed_events", S_IRUSR, root,
226 (u32 *)&host->completed_events);
227 if (!node)
228 goto err;
229
230 return;
231
232 err:
233 dev_err(&mmc->class_dev, "failed to initialize debugfs for slot\n");
234 }
235 #endif /* defined(CONFIG_DEBUG_FS) */
236
237 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset)
238 {
239 u32 ctrl;
240
241 ctrl = mci_readl(host, CTRL);
242 ctrl |= reset;
243 mci_writel(host, CTRL, ctrl);
244
245 /* wait till resets clear */
246 if (readl_poll_timeout_atomic(host->regs + SDMMC_CTRL, ctrl,
247 !(ctrl & reset),
248 1, 500 * USEC_PER_MSEC)) {
249 dev_err(host->dev,
250 "Timeout resetting block (ctrl reset %#x)\n",
251 ctrl & reset);
252 return false;
253 }
254
255 return true;
256 }
257
258 static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags)
259 {
260 u32 status;
261
262 /*
263 * Databook says that before issuing a new data transfer command
264 * we need to check to see if the card is busy. Data transfer commands
265 * all have SDMMC_CMD_PRV_DAT_WAIT set, so we'll key off that.
266 *
267 * ...also allow sending for SDMMC_CMD_VOLT_SWITCH where busy is
268 * expected.
269 */
270 if ((cmd_flags & SDMMC_CMD_PRV_DAT_WAIT) &&
271 !(cmd_flags & SDMMC_CMD_VOLT_SWITCH)) {
272 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS,
273 status,
274 !(status & SDMMC_STATUS_BUSY),
275 10, 500 * USEC_PER_MSEC))
276 dev_err(host->dev, "Busy; trying anyway\n");
277 }
278 }
279
280 static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)
281 {
282 struct dw_mci *host = slot->host;
283 unsigned int cmd_status = 0;
284
285 mci_writel(host, CMDARG, arg);
286 wmb(); /* drain writebuffer */
287 dw_mci_wait_while_busy(host, cmd);
288 mci_writel(host, CMD, SDMMC_CMD_START | cmd);
289
290 if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status,
291 !(cmd_status & SDMMC_CMD_START),
292 1, 500 * USEC_PER_MSEC))
293 dev_err(&slot->mmc->class_dev,
294 "Timeout sending command (cmd %#x arg %#x status %#x)\n",
295 cmd, arg, cmd_status);
296 }
297
298 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
299 {
300 struct dw_mci_slot *slot = mmc_priv(mmc);
301 struct dw_mci *host = slot->host;
302 u32 cmdr;
303
304 cmd->error = -EINPROGRESS;
305 cmdr = cmd->opcode;
306
307 if (cmd->opcode == MMC_STOP_TRANSMISSION ||
308 cmd->opcode == MMC_GO_IDLE_STATE ||
309 cmd->opcode == MMC_GO_INACTIVE_STATE ||
310 (cmd->opcode == SD_IO_RW_DIRECT &&
311 ((cmd->arg >> 9) & 0x1FFFF) == SDIO_CCCR_ABORT))
312 cmdr |= SDMMC_CMD_STOP;
313 else if (cmd->opcode != MMC_SEND_STATUS && cmd->data)
314 cmdr |= SDMMC_CMD_PRV_DAT_WAIT;
315
316 if (cmd->opcode == SD_SWITCH_VOLTAGE) {
317 u32 clk_en_a;
318
319 /* Special bit makes CMD11 not die */
320 cmdr |= SDMMC_CMD_VOLT_SWITCH;
321
322 /* Change state to continue to handle CMD11 weirdness */
323 WARN_ON(slot->host->state != STATE_SENDING_CMD);
324 slot->host->state = STATE_SENDING_CMD11;
325
326 /*
327 * We need to disable low power mode (automatic clock stop)
328 * while doing voltage switch so we don't confuse the card,
329 * since stopping the clock is a specific part of the UHS
330 * voltage change dance.
331 *
332 * Note that low power mode (SDMMC_CLKEN_LOW_PWR) will be
333 * unconditionally turned back on in dw_mci_setup_bus() if it's
334 * ever called with a non-zero clock. That shouldn't happen
335 * until the voltage change is all done.
336 */
337 clk_en_a = mci_readl(host, CLKENA);
338 clk_en_a &= ~(SDMMC_CLKEN_LOW_PWR << slot->id);
339 mci_writel(host, CLKENA, clk_en_a);
340 mci_send_cmd(slot, SDMMC_CMD_UPD_CLK |
341 SDMMC_CMD_PRV_DAT_WAIT, 0);
342 }
343
344 if (cmd->flags & MMC_RSP_PRESENT) {
345 /* We expect a response, so set this bit */
346 cmdr |= SDMMC_CMD_RESP_EXP;
347 if (cmd->flags & MMC_RSP_136)
348 cmdr |= SDMMC_CMD_RESP_LONG;
349 }
350
351 if (cmd->flags & MMC_RSP_CRC)
352 cmdr |= SDMMC_CMD_RESP_CRC;
353
354 if (cmd->data) {
355 cmdr |= SDMMC_CMD_DAT_EXP;
356 if (cmd->data->flags & MMC_DATA_WRITE)
357 cmdr |= SDMMC_CMD_DAT_WR;
358 }
359
360 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &slot->flags))
361 cmdr |= SDMMC_CMD_USE_HOLD_REG;
362
363 return cmdr;
364 }
365
366 static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)
367 {
368 struct mmc_command *stop;
369 u32 cmdr;
370
371 if (!cmd->data)
372 return 0;
373
374 stop = &host->stop_abort;
375 cmdr = cmd->opcode;
376 memset(stop, 0, sizeof(struct mmc_command));
377
378 if (cmdr == MMC_READ_SINGLE_BLOCK ||
379 cmdr == MMC_READ_MULTIPLE_BLOCK ||
380 cmdr == MMC_WRITE_BLOCK ||
381 cmdr == MMC_WRITE_MULTIPLE_BLOCK ||
382 cmdr == MMC_SEND_TUNING_BLOCK ||
383 cmdr == MMC_SEND_TUNING_BLOCK_HS200) {
384 stop->opcode = MMC_STOP_TRANSMISSION;
385 stop->arg = 0;
386 stop->flags = MMC_RSP_R1B | MMC_CMD_AC;
387 } else if (cmdr == SD_IO_RW_EXTENDED) {
388 stop->opcode = SD_IO_RW_DIRECT;
389 stop->arg |= (1 << 31) | (0 << 28) | (SDIO_CCCR_ABORT << 9) |
390 ((cmd->arg >> 28) & 0x7);
391 stop->flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_AC;
392 } else {
393 return 0;
394 }
395
396 cmdr = stop->opcode | SDMMC_CMD_STOP |
397 SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP;
398
399 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags))
400 cmdr |= SDMMC_CMD_USE_HOLD_REG;
401
402 return cmdr;
403 }
404
405 static inline void dw_mci_set_cto(struct dw_mci *host)
406 {
407 unsigned int cto_clks;
408 unsigned int cto_div;
409 unsigned int cto_ms;
410 unsigned long irqflags;
411
412 cto_clks = mci_readl(host, TMOUT) & 0xff;
413 cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;
414 if (cto_div == 0)
415 cto_div = 1;
416 cto_ms = DIV_ROUND_UP(MSEC_PER_SEC * cto_clks * cto_div, host->bus_hz);
417
418 /* add a bit spare time */
419 cto_ms += 10;
420
421 /*
422 * The durations we're working with are fairly short so we have to be
423 * extra careful about synchronization here. Specifically in hardware a
424 * command timeout is _at most_ 5.1 ms, so that means we expect an
425 * interrupt (either command done or timeout) to come rather quickly
426 * after the mci_writel. ...but just in case we have a long interrupt
427 * latency let's add a bit of paranoia.
428 *
429 * In general we'll assume that at least an interrupt will be asserted
430 * in hardware by the time the cto_timer runs. ...and if it hasn't
431 * been asserted in hardware by that time then we'll assume it'll never
432 * come.
433 */
434 spin_lock_irqsave(&host->irq_lock, irqflags);
435 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
436 mod_timer(&host->cto_timer,
437 jiffies + msecs_to_jiffies(cto_ms) + 1);
438 spin_unlock_irqrestore(&host->irq_lock, irqflags);
439 }
440
441 static void dw_mci_start_command(struct dw_mci *host,
442 struct mmc_command *cmd, u32 cmd_flags)
443 {
444 host->cmd = cmd;
445 dev_vdbg(host->dev,
446 "start command: ARGR=0x%08x CMDR=0x%08x\n",
447 cmd->arg, cmd_flags);
448
449 mci_writel(host, CMDARG, cmd->arg);
450 wmb(); /* drain writebuffer */
451 dw_mci_wait_while_busy(host, cmd_flags);
452
453 mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START);
454
455 /* response expected command only */
456 if (cmd_flags & SDMMC_CMD_RESP_EXP)
457 dw_mci_set_cto(host);
458 }
459
460 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data)
461 {
462 struct mmc_command *stop = &host->stop_abort;
463
464 dw_mci_start_command(host, stop, host->stop_cmdr);
465 }
466
467 /* DMA interface functions */
468 static void dw_mci_stop_dma(struct dw_mci *host)
469 {
470 if (host->using_dma) {
471 host->dma_ops->stop(host);
472 host->dma_ops->cleanup(host);
473 }
474
475 /* Data transfer was stopped by the interrupt handler */
476 set_bit(EVENT_XFER_COMPLETE, &host->pending_events);
477 }
478
479 static void dw_mci_dma_cleanup(struct dw_mci *host)
480 {
481 struct mmc_data *data = host->data;
482
483 if (data && data->host_cookie == COOKIE_MAPPED) {
484 dma_unmap_sg(host->dev,
485 data->sg,
486 data->sg_len,
487 mmc_get_dma_dir(data));
488 data->host_cookie = COOKIE_UNMAPPED;
489 }
490 }
491
492 static void dw_mci_idmac_reset(struct dw_mci *host)
493 {
494 u32 bmod = mci_readl(host, BMOD);
495 /* Software reset of DMA */
496 bmod |= SDMMC_IDMAC_SWRESET;
497 mci_writel(host, BMOD, bmod);
498 }
499
500 static void dw_mci_idmac_stop_dma(struct dw_mci *host)
501 {
502 u32 temp;
503
504 /* Disable and reset the IDMAC interface */
505 temp = mci_readl(host, CTRL);
506 temp &= ~SDMMC_CTRL_USE_IDMAC;
507 temp |= SDMMC_CTRL_DMA_RESET;
508 mci_writel(host, CTRL, temp);
509
510 /* Stop the IDMAC running */
511 temp = mci_readl(host, BMOD);
512 temp &= ~(SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB);
513 temp |= SDMMC_IDMAC_SWRESET;
514 mci_writel(host, BMOD, temp);
515 }
516
517 static void dw_mci_dmac_complete_dma(void *arg)
518 {
519 struct dw_mci *host = arg;
520 struct mmc_data *data = host->data;
521
522 dev_vdbg(host->dev, "DMA complete\n");
523
524 if ((host->use_dma == TRANS_MODE_EDMAC) &&
525 data && (data->flags & MMC_DATA_READ))
526 /* Invalidate cache after read */
527 dma_sync_sg_for_cpu(mmc_dev(host->slot->mmc),
528 data->sg,
529 data->sg_len,
530 DMA_FROM_DEVICE);
531
532 host->dma_ops->cleanup(host);
533
534 /*
535 * If the card was removed, data will be NULL. No point in trying to
536 * send the stop command or waiting for NBUSY in this case.
537 */
538 if (data) {
539 set_bit(EVENT_XFER_COMPLETE, &host->pending_events);
540 tasklet_schedule(&host->tasklet);
541 }
542 }
543
544 static int dw_mci_idmac_init(struct dw_mci *host)
545 {
546 int i;
547
548 if (host->dma_64bit_address == 1) {
549 struct idmac_desc_64addr *p;
550 /* Number of descriptors in the ring buffer */
551 host->ring_size =
552 DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr);
553
554 /* Forward link the descriptor list */
555 for (i = 0, p = host->sg_cpu; i < host->ring_size - 1;
556 i++, p++) {
557 p->des6 = (host->sg_dma +
558 (sizeof(struct idmac_desc_64addr) *
559 (i + 1))) & 0xffffffff;
560
561 p->des7 = (u64)(host->sg_dma +
562 (sizeof(struct idmac_desc_64addr) *
563 (i + 1))) >> 32;
564 /* Initialize reserved and buffer size fields to "0" */
565 p->des1 = 0;
566 p->des2 = 0;
567 p->des3 = 0;
568 }
569
570 /* Set the last descriptor as the end-of-ring descriptor */
571 p->des6 = host->sg_dma & 0xffffffff;
572 p->des7 = (u64)host->sg_dma >> 32;
573 p->des0 = IDMAC_DES0_ER;
574
575 } else {
576 struct idmac_desc *p;
577 /* Number of descriptors in the ring buffer */
578 host->ring_size =
579 DESC_RING_BUF_SZ / sizeof(struct idmac_desc);
580
581 /* Forward link the descriptor list */
582 for (i = 0, p = host->sg_cpu;
583 i < host->ring_size - 1;
584 i++, p++) {
585 p->des3 = cpu_to_le32(host->sg_dma +
586 (sizeof(struct idmac_desc) * (i + 1)));
587 p->des1 = 0;
588 }
589
590 /* Set the last descriptor as the end-of-ring descriptor */
591 p->des3 = cpu_to_le32(host->sg_dma);
592 p->des0 = cpu_to_le32(IDMAC_DES0_ER);
593 }
594
595 dw_mci_idmac_reset(host);
596
597 if (host->dma_64bit_address == 1) {
598 /* Mask out interrupts - get Tx & Rx complete only */
599 mci_writel(host, IDSTS64, IDMAC_INT_CLR);
600 mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI |
601 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI);
602
603 /* Set the descriptor base address */
604 mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff);
605 mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32);
606
607 } else {
608 /* Mask out interrupts - get Tx & Rx complete only */
609 mci_writel(host, IDSTS, IDMAC_INT_CLR);
610 mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI |
611 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI);
612
613 /* Set the descriptor base address */
614 mci_writel(host, DBADDR, host->sg_dma);
615 }
616
617 return 0;
618 }
619
620 static inline int dw_mci_prepare_desc64(struct dw_mci *host,
621 struct mmc_data *data,
622 unsigned int sg_len)
623 {
624 unsigned int desc_len;
625 struct idmac_desc_64addr *desc_first, *desc_last, *desc;
626 u32 val;
627 int i;
628
629 desc_first = desc_last = desc = host->sg_cpu;
630
631 for (i = 0; i < sg_len; i++) {
632 unsigned int length = sg_dma_len(&data->sg[i]);
633
634 u64 mem_addr = sg_dma_address(&data->sg[i]);
635
636 for ( ; length ; desc++) {
637 desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
638 length : DW_MCI_DESC_DATA_LENGTH;
639
640 length -= desc_len;
641
642 /*
643 * Wait for the former clear OWN bit operation
644 * of IDMAC to make sure that this descriptor
645 * isn't still owned by IDMAC as IDMAC's write
646 * ops and CPU's read ops are asynchronous.
647 */
648 if (readl_poll_timeout_atomic(&desc->des0, val,
649 !(val & IDMAC_DES0_OWN),
650 10, 100 * USEC_PER_MSEC))
651 goto err_own_bit;
652
653 /*
654 * Set the OWN bit and disable interrupts
655 * for this descriptor
656 */
657 desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |
658 IDMAC_DES0_CH;
659
660 /* Buffer length */
661 IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);
662
663 /* Physical address to DMA to/from */
664 desc->des4 = mem_addr & 0xffffffff;
665 desc->des5 = mem_addr >> 32;
666
667 /* Update physical address for the next desc */
668 mem_addr += desc_len;
669
670 /* Save pointer to the last descriptor */
671 desc_last = desc;
672 }
673 }
674
675 /* Set first descriptor */
676 desc_first->des0 |= IDMAC_DES0_FD;
677
678 /* Set last descriptor */
679 desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
680 desc_last->des0 |= IDMAC_DES0_LD;
681
682 return 0;
683 err_own_bit:
684 /* restore the descriptor chain as it's polluted */
685 dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
686 memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
687 dw_mci_idmac_init(host);
688 return -EINVAL;
689 }
690
691
692 static inline int dw_mci_prepare_desc32(struct dw_mci *host,
693 struct mmc_data *data,
694 unsigned int sg_len)
695 {
696 unsigned int desc_len;
697 struct idmac_desc *desc_first, *desc_last, *desc;
698 u32 val;
699 int i;
700
701 desc_first = desc_last = desc = host->sg_cpu;
702
703 for (i = 0; i < sg_len; i++) {
704 unsigned int length = sg_dma_len(&data->sg[i]);
705
706 u32 mem_addr = sg_dma_address(&data->sg[i]);
707
708 for ( ; length ; desc++) {
709 desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
710 length : DW_MCI_DESC_DATA_LENGTH;
711
712 length -= desc_len;
713
714 /*
715 * Wait for the former clear OWN bit operation
716 * of IDMAC to make sure that this descriptor
717 * isn't still owned by IDMAC as IDMAC's write
718 * ops and CPU's read ops are asynchronous.
719 */
720 if (readl_poll_timeout_atomic(&desc->des0, val,
721 IDMAC_OWN_CLR64(val),
722 10,
723 100 * USEC_PER_MSEC))
724 goto err_own_bit;
725
726 /*
727 * Set the OWN bit and disable interrupts
728 * for this descriptor
729 */
730 desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |
731 IDMAC_DES0_DIC |
732 IDMAC_DES0_CH);
733
734 /* Buffer length */
735 IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
736
737 /* Physical address to DMA to/from */
738 desc->des2 = cpu_to_le32(mem_addr);
739
740 /* Update physical address for the next desc */
741 mem_addr += desc_len;
742
743 /* Save pointer to the last descriptor */
744 desc_last = desc;
745 }
746 }
747
748 /* Set first descriptor */
749 desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
750
751 /* Set last descriptor */
752 desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |
753 IDMAC_DES0_DIC));
754 desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
755
756 return 0;
757 err_own_bit:
758 /* restore the descriptor chain as it's polluted */
759 dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
760 memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
761 dw_mci_idmac_init(host);
762 return -EINVAL;
763 }
764
765 static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)
766 {
767 u32 temp;
768 int ret;
769
770 if (host->dma_64bit_address == 1)
771 ret = dw_mci_prepare_desc64(host, host->data, sg_len);
772 else
773 ret = dw_mci_prepare_desc32(host, host->data, sg_len);
774
775 if (ret)
776 goto out;
777
778 /* drain writebuffer */
779 wmb();
780
781 /* Make sure to reset DMA in case we did PIO before this */
782 dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET);
783 dw_mci_idmac_reset(host);
784
785 /* Select IDMAC interface */
786 temp = mci_readl(host, CTRL);
787 temp |= SDMMC_CTRL_USE_IDMAC;
788 mci_writel(host, CTRL, temp);
789
790 /* drain writebuffer */
791 wmb();
792
793 /* Enable the IDMAC */
794 temp = mci_readl(host, BMOD);
795 temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB;
796 mci_writel(host, BMOD, temp);
797
798 /* Start it running */
799 mci_writel(host, PLDMND, 1);
800
801 out:
802 return ret;
803 }
804
805 static const struct dw_mci_dma_ops dw_mci_idmac_ops = {
806 .init = dw_mci_idmac_init,
807 .start = dw_mci_idmac_start_dma,
808 .stop = dw_mci_idmac_stop_dma,
809 .complete = dw_mci_dmac_complete_dma,
810 .cleanup = dw_mci_dma_cleanup,
811 };
812
813 static void dw_mci_edmac_stop_dma(struct dw_mci *host)
814 {
815 dmaengine_terminate_async(host->dms->ch);
816 }
817
818 static int dw_mci_edmac_start_dma(struct dw_mci *host,
819 unsigned int sg_len)
820 {
821 struct dma_slave_config cfg;
822 struct dma_async_tx_descriptor *desc = NULL;
823 struct scatterlist *sgl = host->data->sg;
824 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256};
825 u32 sg_elems = host->data->sg_len;
826 u32 fifoth_val;
827 u32 fifo_offset = host->fifo_reg - host->regs;
828 int ret = 0;
829
830 /* Set external dma config: burst size, burst width */
831 cfg.dst_addr = host->phy_regs + fifo_offset;
832 cfg.src_addr = cfg.dst_addr;
833 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
834 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
835
836 /* Match burst msize with external dma config */
837 fifoth_val = mci_readl(host, FIFOTH);
838 cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7];
839 cfg.src_maxburst = cfg.dst_maxburst;
840
841 if (host->data->flags & MMC_DATA_WRITE)
842 cfg.direction = DMA_MEM_TO_DEV;
843 else
844 cfg.direction = DMA_DEV_TO_MEM;
845
846 ret = dmaengine_slave_config(host->dms->ch, &cfg);
847 if (ret) {
848 dev_err(host->dev, "Failed to config edmac.\n");
849 return -EBUSY;
850 }
851
852 desc = dmaengine_prep_slave_sg(host->dms->ch, sgl,
853 sg_len, cfg.direction,
854 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
855 if (!desc) {
856 dev_err(host->dev, "Can't prepare slave sg.\n");
857 return -EBUSY;
858 }
859
860 /* Set dw_mci_dmac_complete_dma as callback */
861 desc->callback = dw_mci_dmac_complete_dma;
862 desc->callback_param = (void *)host;
863 dmaengine_submit(desc);
864
865 /* Flush cache before write */
866 if (host->data->flags & MMC_DATA_WRITE)
867 dma_sync_sg_for_device(mmc_dev(host->slot->mmc), sgl,
868 sg_elems, DMA_TO_DEVICE);
869
870 dma_async_issue_pending(host->dms->ch);
871
872 return 0;
873 }
874
875 static int dw_mci_edmac_init(struct dw_mci *host)
876 {
877 /* Request external dma channel */
878 host->dms = kzalloc(sizeof(struct dw_mci_dma_slave), GFP_KERNEL);
879 if (!host->dms)
880 return -ENOMEM;
881
882 host->dms->ch = dma_request_slave_channel(host->dev, "rx-tx");
883 if (!host->dms->ch) {
884 dev_err(host->dev, "Failed to get external DMA channel.\n");
885 kfree(host->dms);
886 host->dms = NULL;
887 return -ENXIO;
888 }
889
890 return 0;
891 }
892
893 static void dw_mci_edmac_exit(struct dw_mci *host)
894 {
895 if (host->dms) {
896 if (host->dms->ch) {
897 dma_release_channel(host->dms->ch);
898 host->dms->ch = NULL;
899 }
900 kfree(host->dms);
901 host->dms = NULL;
902 }
903 }
904
905 static const struct dw_mci_dma_ops dw_mci_edmac_ops = {
906 .init = dw_mci_edmac_init,
907 .exit = dw_mci_edmac_exit,
908 .start = dw_mci_edmac_start_dma,
909 .stop = dw_mci_edmac_stop_dma,
910 .complete = dw_mci_dmac_complete_dma,
911 .cleanup = dw_mci_dma_cleanup,
912 };
913
914 static int dw_mci_pre_dma_transfer(struct dw_mci *host,
915 struct mmc_data *data,
916 int cookie)
917 {
918 struct scatterlist *sg;
919 unsigned int i, sg_len;
920
921 if (data->host_cookie == COOKIE_PRE_MAPPED)
922 return data->sg_len;
923
924 /*
925 * We don't do DMA on "complex" transfers, i.e. with
926 * non-word-aligned buffers or lengths. Also, we don't bother
927 * with all the DMA setup overhead for short transfers.
928 */
929 if (data->blocks * data->blksz < DW_MCI_DMA_THRESHOLD)
930 return -EINVAL;
931
932 if (data->blksz & 3)
933 return -EINVAL;
934
935 for_each_sg(data->sg, sg, data->sg_len, i) {
936 if (sg->offset & 3 || sg->length & 3)
937 return -EINVAL;
938 }
939
940 sg_len = dma_map_sg(host->dev,
941 data->sg,
942 data->sg_len,
943 mmc_get_dma_dir(data));
944 if (sg_len == 0)
945 return -EINVAL;
946
947 data->host_cookie = cookie;
948
949 return sg_len;
950 }
951
952 static void dw_mci_pre_req(struct mmc_host *mmc,
953 struct mmc_request *mrq)
954 {
955 struct dw_mci_slot *slot = mmc_priv(mmc);
956 struct mmc_data *data = mrq->data;
957
958 if (!slot->host->use_dma || !data)
959 return;
960
961 /* This data might be unmapped at this time */
962 data->host_cookie = COOKIE_UNMAPPED;
963
964 if (dw_mci_pre_dma_transfer(slot->host, mrq->data,
965 COOKIE_PRE_MAPPED) < 0)
966 data->host_cookie = COOKIE_UNMAPPED;
967 }
968
969 static void dw_mci_post_req(struct mmc_host *mmc,
970 struct mmc_request *mrq,
971 int err)
972 {
973 struct dw_mci_slot *slot = mmc_priv(mmc);
974 struct mmc_data *data = mrq->data;
975
976 if (!slot->host->use_dma || !data)
977 return;
978
979 if (data->host_cookie != COOKIE_UNMAPPED)
980 dma_unmap_sg(slot->host->dev,
981 data->sg,
982 data->sg_len,
983 mmc_get_dma_dir(data));
984 data->host_cookie = COOKIE_UNMAPPED;
985 }
986
987 static int dw_mci_get_cd(struct mmc_host *mmc)
988 {
989 int present;
990 struct dw_mci_slot *slot = mmc_priv(mmc);
991 struct dw_mci *host = slot->host;
992 int gpio_cd = mmc_gpio_get_cd(mmc);
993
994 /* Use platform get_cd function, else try onboard card detect */
995 if (((mmc->caps & MMC_CAP_NEEDS_POLL)
996 || !mmc_card_is_removable(mmc))) {
997 present = 1;
998
999 if (!test_bit(DW_MMC_CARD_PRESENT, &slot->flags)) {
1000 if (mmc->caps & MMC_CAP_NEEDS_POLL) {
1001 dev_info(&mmc->class_dev,
1002 "card is polling.\n");
1003 } else {
1004 dev_info(&mmc->class_dev,
1005 "card is non-removable.\n");
1006 }
1007 set_bit(DW_MMC_CARD_PRESENT, &slot->flags);
1008 }
1009
1010 return present;
1011 } else if (gpio_cd >= 0)
1012 present = gpio_cd;
1013 else
1014 present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))
1015 == 0 ? 1 : 0;
1016
1017 spin_lock_bh(&host->lock);
1018 if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &slot->flags))
1019 dev_dbg(&mmc->class_dev, "card is present\n");
1020 else if (!present &&
1021 !test_and_clear_bit(DW_MMC_CARD_PRESENT, &slot->flags))
1022 dev_dbg(&mmc->class_dev, "card is not present\n");
1023 spin_unlock_bh(&host->lock);
1024
1025 return present;
1026 }
1027
1028 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)
1029 {
1030 unsigned int blksz = data->blksz;
1031 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256};
1032 u32 fifo_width = 1 << host->data_shift;
1033 u32 blksz_depth = blksz / fifo_width, fifoth_val;
1034 u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers;
1035 int idx = ARRAY_SIZE(mszs) - 1;
1036
1037 /* pio should ship this scenario */
1038 if (!host->use_dma)
1039 return;
1040
1041 tx_wmark = (host->fifo_depth) / 2;
1042 tx_wmark_invers = host->fifo_depth - tx_wmark;
1043
1044 /*
1045 * MSIZE is '1',
1046 * if blksz is not a multiple of the FIFO width
1047 */
1048 if (blksz % fifo_width)
1049 goto done;
1050
1051 do {
1052 if (!((blksz_depth % mszs[idx]) ||
1053 (tx_wmark_invers % mszs[idx]))) {
1054 msize = idx;
1055 rx_wmark = mszs[idx] - 1;
1056 break;
1057 }
1058 } while (--idx > 0);
1059 /*
1060 * If idx is '0', it won't be tried
1061 * Thus, initial values are uesed
1062 */
1063 done:
1064 fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark);
1065 mci_writel(host, FIFOTH, fifoth_val);
1066 }
1067
1068 static void dw_mci_ctrl_thld(struct dw_mci *host, struct mmc_data *data)
1069 {
1070 unsigned int blksz = data->blksz;
1071 u32 blksz_depth, fifo_depth;
1072 u16 thld_size;
1073 u8 enable;
1074
1075 /*
1076 * CDTHRCTL doesn't exist prior to 240A (in fact that register offset is
1077 * in the FIFO region, so we really shouldn't access it).
1078 */
1079 if (host->verid < DW_MMC_240A ||
1080 (host->verid < DW_MMC_280A && data->flags & MMC_DATA_WRITE))
1081 return;
1082
1083 /*
1084 * Card write Threshold is introduced since 2.80a
1085 * It's used when HS400 mode is enabled.
1086 */
1087 if (data->flags & MMC_DATA_WRITE &&
1088 !(host->timing != MMC_TIMING_MMC_HS400))
1089 return;
1090
1091 if (data->flags & MMC_DATA_WRITE)
1092 enable = SDMMC_CARD_WR_THR_EN;
1093 else
1094 enable = SDMMC_CARD_RD_THR_EN;
1095
1096 if (host->timing != MMC_TIMING_MMC_HS200 &&
1097 host->timing != MMC_TIMING_UHS_SDR104)
1098 goto disable;
1099
1100 blksz_depth = blksz / (1 << host->data_shift);
1101 fifo_depth = host->fifo_depth;
1102
1103 if (blksz_depth > fifo_depth)
1104 goto disable;
1105
1106 /*
1107 * If (blksz_depth) >= (fifo_depth >> 1), should be 'thld_size <= blksz'
1108 * If (blksz_depth) < (fifo_depth >> 1), should be thld_size = blksz
1109 * Currently just choose blksz.
1110 */
1111 thld_size = blksz;
1112 mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(thld_size, enable));
1113 return;
1114
1115 disable:
1116 mci_writel(host, CDTHRCTL, 0);
1117 }
1118
1119 static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data)
1120 {
1121 unsigned long irqflags;
1122 int sg_len;
1123 u32 temp;
1124
1125 host->using_dma = 0;
1126
1127 /* If we don't have a channel, we can't do DMA */
1128 if (!host->use_dma)
1129 return -ENODEV;
1130
1131 sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED);
1132 if (sg_len < 0) {
1133 host->dma_ops->stop(host);
1134 return sg_len;
1135 }
1136
1137 host->using_dma = 1;
1138
1139 if (host->use_dma == TRANS_MODE_IDMAC)
1140 dev_vdbg(host->dev,
1141 "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n",
1142 (unsigned long)host->sg_cpu,
1143 (unsigned long)host->sg_dma,
1144 sg_len);
1145
1146 /*
1147 * Decide the MSIZE and RX/TX Watermark.
1148 * If current block size is same with previous size,
1149 * no need to update fifoth.
1150 */
1151 if (host->prev_blksz != data->blksz)
1152 dw_mci_adjust_fifoth(host, data);
1153
1154 /* Enable the DMA interface */
1155 temp = mci_readl(host, CTRL);
1156 temp |= SDMMC_CTRL_DMA_ENABLE;
1157 mci_writel(host, CTRL, temp);
1158
1159 /* Disable RX/TX IRQs, let DMA handle it */
1160 spin_lock_irqsave(&host->irq_lock, irqflags);
1161 temp = mci_readl(host, INTMASK);
1162 temp &= ~(SDMMC_INT_RXDR | SDMMC_INT_TXDR);
1163 mci_writel(host, INTMASK, temp);
1164 spin_unlock_irqrestore(&host->irq_lock, irqflags);
1165
1166 if (host->dma_ops->start(host, sg_len)) {
1167 host->dma_ops->stop(host);
1168 /* We can't do DMA, try PIO for this one */
1169 dev_dbg(host->dev,
1170 "%s: fall back to PIO mode for current transfer\n",
1171 __func__);
1172 return -ENODEV;
1173 }
1174
1175 return 0;
1176 }
1177
1178 static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)
1179 {
1180 unsigned long irqflags;
1181 int flags = SG_MITER_ATOMIC;
1182 u32 temp;
1183
1184 data->error = -EINPROGRESS;
1185
1186 WARN_ON(host->data);
1187 host->sg = NULL;
1188 host->data = data;
1189
1190 if (data->flags & MMC_DATA_READ)
1191 host->dir_status = DW_MCI_RECV_STATUS;
1192 else
1193 host->dir_status = DW_MCI_SEND_STATUS;
1194
1195 dw_mci_ctrl_thld(host, data);
1196
1197 if (dw_mci_submit_data_dma(host, data)) {
1198 if (host->data->flags & MMC_DATA_READ)
1199 flags |= SG_MITER_TO_SG;
1200 else
1201 flags |= SG_MITER_FROM_SG;
1202
1203 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1204 host->sg = data->sg;
1205 host->part_buf_start = 0;
1206 host->part_buf_count = 0;
1207
1208 mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR);
1209
1210 spin_lock_irqsave(&host->irq_lock, irqflags);
1211 temp = mci_readl(host, INTMASK);
1212 temp |= SDMMC_INT_TXDR | SDMMC_INT_RXDR;
1213 mci_writel(host, INTMASK, temp);
1214 spin_unlock_irqrestore(&host->irq_lock, irqflags);
1215
1216 temp = mci_readl(host, CTRL);
1217 temp &= ~SDMMC_CTRL_DMA_ENABLE;
1218 mci_writel(host, CTRL, temp);
1219
1220 /*
1221 * Use the initial fifoth_val for PIO mode. If wm_algined
1222 * is set, we set watermark same as data size.
1223 * If next issued data may be transfered by DMA mode,
1224 * prev_blksz should be invalidated.
1225 */
1226 if (host->wm_aligned)
1227 dw_mci_adjust_fifoth(host, data);
1228 else
1229 mci_writel(host, FIFOTH, host->fifoth_val);
1230 host->prev_blksz = 0;
1231 } else {
1232 /*
1233 * Keep the current block size.
1234 * It will be used to decide whether to update
1235 * fifoth register next time.
1236 */
1237 host->prev_blksz = data->blksz;
1238 }
1239 }
1240
1241 static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
1242 {
1243 struct dw_mci *host = slot->host;
1244 unsigned int clock = slot->clock;
1245 u32 div;
1246 u32 clk_en_a;
1247 u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT;
1248
1249 /* We must continue to set bit 28 in CMD until the change is complete */
1250 if (host->state == STATE_WAITING_CMD11_DONE)
1251 sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH;
1252
1253 if (!clock) {
1254 mci_writel(host, CLKENA, 0);
1255 mci_send_cmd(slot, sdmmc_cmd_bits, 0);
1256 } else if (clock != host->current_speed || force_clkinit) {
1257 div = host->bus_hz / clock;
1258 if (host->bus_hz % clock && host->bus_hz > clock)
1259 /*
1260 * move the + 1 after the divide to prevent
1261 * over-clocking the card.
1262 */
1263 div += 1;
1264
1265 div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;
1266
1267 if ((clock != slot->__clk_old &&
1268 !test_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags)) ||
1269 force_clkinit) {
1270 /* Silent the verbose log if calling from PM context */
1271 if (!force_clkinit)
1272 dev_info(&slot->mmc->class_dev,
1273 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
1274 slot->id, host->bus_hz, clock,
1275 div ? ((host->bus_hz / div) >> 1) :
1276 host->bus_hz, div);
1277
1278 /*
1279 * If card is polling, display the message only
1280 * one time at boot time.
1281 */
1282 if (slot->mmc->caps & MMC_CAP_NEEDS_POLL &&
1283 slot->mmc->f_min == clock)
1284 set_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags);
1285 }
1286
1287 /* disable clock */
1288 mci_writel(host, CLKENA, 0);
1289 mci_writel(host, CLKSRC, 0);
1290
1291 /* inform CIU */
1292 mci_send_cmd(slot, sdmmc_cmd_bits, 0);
1293
1294 /* set clock to desired speed */
1295 mci_writel(host, CLKDIV, div);
1296
1297 /* inform CIU */
1298 mci_send_cmd(slot, sdmmc_cmd_bits, 0);
1299
1300 /* enable clock; only low power if no SDIO */
1301 clk_en_a = SDMMC_CLKEN_ENABLE << slot->id;
1302 if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags))
1303 clk_en_a |= SDMMC_CLKEN_LOW_PWR << slot->id;
1304 mci_writel(host, CLKENA, clk_en_a);
1305
1306 /* inform CIU */
1307 mci_send_cmd(slot, sdmmc_cmd_bits, 0);
1308
1309 /* keep the last clock value that was requested from core */
1310 slot->__clk_old = clock;
1311 }
1312
1313 host->current_speed = clock;
1314
1315 /* Set the current slot bus width */
1316 mci_writel(host, CTYPE, (slot->ctype << slot->id));
1317 }
1318
1319 static void __dw_mci_start_request(struct dw_mci *host,
1320 struct dw_mci_slot *slot,
1321 struct mmc_command *cmd)
1322 {
1323 struct mmc_request *mrq;
1324 struct mmc_data *data;
1325 u32 cmdflags;
1326
1327 mrq = slot->mrq;
1328
1329 host->mrq = mrq;
1330
1331 host->pending_events = 0;
1332 host->completed_events = 0;
1333 host->cmd_status = 0;
1334 host->data_status = 0;
1335 host->dir_status = 0;
1336
1337 data = cmd->data;
1338 if (data) {
1339 mci_writel(host, TMOUT, 0xFFFFFFFF);
1340 mci_writel(host, BYTCNT, data->blksz*data->blocks);
1341 mci_writel(host, BLKSIZ, data->blksz);
1342 }
1343
1344 cmdflags = dw_mci_prepare_command(slot->mmc, cmd);
1345
1346 /* this is the first command, send the initialization clock */
1347 if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &slot->flags))
1348 cmdflags |= SDMMC_CMD_INIT;
1349
1350 if (data) {
1351 dw_mci_submit_data(host, data);
1352 wmb(); /* drain writebuffer */
1353 }
1354
1355 dw_mci_start_command(host, cmd, cmdflags);
1356
1357 if (cmd->opcode == SD_SWITCH_VOLTAGE) {
1358 unsigned long irqflags;
1359
1360 /*
1361 * Databook says to fail after 2ms w/ no response, but evidence
1362 * shows that sometimes the cmd11 interrupt takes over 130ms.
1363 * We'll set to 500ms, plus an extra jiffy just in case jiffies
1364 * is just about to roll over.
1365 *
1366 * We do this whole thing under spinlock and only if the
1367 * command hasn't already completed (indicating the the irq
1368 * already ran so we don't want the timeout).
1369 */
1370 spin_lock_irqsave(&host->irq_lock, irqflags);
1371 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
1372 mod_timer(&host->cmd11_timer,
1373 jiffies + msecs_to_jiffies(500) + 1);
1374 spin_unlock_irqrestore(&host->irq_lock, irqflags);
1375 }
1376
1377 host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);
1378 }
1379
1380 static void dw_mci_start_request(struct dw_mci *host,
1381 struct dw_mci_slot *slot)
1382 {
1383 struct mmc_request *mrq = slot->mrq;
1384 struct mmc_command *cmd;
1385
1386 cmd = mrq->sbc ? mrq->sbc : mrq->cmd;
1387 __dw_mci_start_request(host, slot, cmd);
1388 }
1389
1390 /* must be called with host->lock held */
1391 static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,
1392 struct mmc_request *mrq)
1393 {
1394 dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
1395 host->state);
1396
1397 slot->mrq = mrq;
1398
1399 if (host->state == STATE_WAITING_CMD11_DONE) {
1400 dev_warn(&slot->mmc->class_dev,
1401 "Voltage change didn't complete\n");
1402 /*
1403 * this case isn't expected to happen, so we can
1404 * either crash here or just try to continue on
1405 * in the closest possible state
1406 */
1407 host->state = STATE_IDLE;
1408 }
1409
1410 if (host->state == STATE_IDLE) {
1411 host->state = STATE_SENDING_CMD;
1412 dw_mci_start_request(host, slot);
1413 } else {
1414 list_add_tail(&slot->queue_node, &host->queue);
1415 }
1416 }
1417
1418 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1419 {
1420 struct dw_mci_slot *slot = mmc_priv(mmc);
1421 struct dw_mci *host = slot->host;
1422
1423 WARN_ON(slot->mrq);
1424
1425 /*
1426 * The check for card presence and queueing of the request must be
1427 * atomic, otherwise the card could be removed in between and the
1428 * request wouldn't fail until another card was inserted.
1429 */
1430
1431 if (!dw_mci_get_cd(mmc)) {
1432 mrq->cmd->error = -ENOMEDIUM;
1433 mmc_request_done(mmc, mrq);
1434 return;
1435 }
1436
1437 spin_lock_bh(&host->lock);
1438
1439 dw_mci_queue_request(host, slot, mrq);
1440
1441 spin_unlock_bh(&host->lock);
1442 }
1443
1444 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1445 {
1446 struct dw_mci_slot *slot = mmc_priv(mmc);
1447 const struct dw_mci_drv_data *drv_data = slot->host->drv_data;
1448 u32 regs;
1449 int ret;
1450
1451 switch (ios->bus_width) {
1452 case MMC_BUS_WIDTH_4:
1453 slot->ctype = SDMMC_CTYPE_4BIT;
1454 break;
1455 case MMC_BUS_WIDTH_8:
1456 slot->ctype = SDMMC_CTYPE_8BIT;
1457 break;
1458 default:
1459 /* set default 1 bit mode */
1460 slot->ctype = SDMMC_CTYPE_1BIT;
1461 }
1462
1463 regs = mci_readl(slot->host, UHS_REG);
1464
1465 /* DDR mode set */
1466 if (ios->timing == MMC_TIMING_MMC_DDR52 ||
1467 ios->timing == MMC_TIMING_UHS_DDR50 ||
1468 ios->timing == MMC_TIMING_MMC_HS400)
1469 regs |= ((0x1 << slot->id) << 16);
1470 else
1471 regs &= ~((0x1 << slot->id) << 16);
1472
1473 mci_writel(slot->host, UHS_REG, regs);
1474 slot->host->timing = ios->timing;
1475
1476 /*
1477 * Use mirror of ios->clock to prevent race with mmc
1478 * core ios update when finding the minimum.
1479 */
1480 slot->clock = ios->clock;
1481
1482 if (drv_data && drv_data->set_ios)
1483 drv_data->set_ios(slot->host, ios);
1484
1485 switch (ios->power_mode) {
1486 case MMC_POWER_UP:
1487 if (!IS_ERR(mmc->supply.vmmc)) {
1488 ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,
1489 ios->vdd);
1490 if (ret) {
1491 dev_err(slot->host->dev,
1492 "failed to enable vmmc regulator\n");
1493 /*return, if failed turn on vmmc*/
1494 return;
1495 }
1496 }
1497 set_bit(DW_MMC_CARD_NEED_INIT, &slot->flags);
1498 regs = mci_readl(slot->host, PWREN);
1499 regs |= (1 << slot->id);
1500 mci_writel(slot->host, PWREN, regs);
1501 break;
1502 case MMC_POWER_ON:
1503 if (!slot->host->vqmmc_enabled) {
1504 if (!IS_ERR(mmc->supply.vqmmc)) {
1505 ret = regulator_enable(mmc->supply.vqmmc);
1506 if (ret < 0)
1507 dev_err(slot->host->dev,
1508 "failed to enable vqmmc\n");
1509 else
1510 slot->host->vqmmc_enabled = true;
1511
1512 } else {
1513 /* Keep track so we don't reset again */
1514 slot->host->vqmmc_enabled = true;
1515 }
1516
1517 /* Reset our state machine after powering on */
1518 dw_mci_ctrl_reset(slot->host,
1519 SDMMC_CTRL_ALL_RESET_FLAGS);
1520 }
1521
1522 /* Adjust clock / bus width after power is up */
1523 dw_mci_setup_bus(slot, false);
1524
1525 break;
1526 case MMC_POWER_OFF:
1527 /* Turn clock off before power goes down */
1528 dw_mci_setup_bus(slot, false);
1529
1530 if (!IS_ERR(mmc->supply.vmmc))
1531 mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1532
1533 if (!IS_ERR(mmc->supply.vqmmc) && slot->host->vqmmc_enabled)
1534 regulator_disable(mmc->supply.vqmmc);
1535 slot->host->vqmmc_enabled = false;
1536
1537 regs = mci_readl(slot->host, PWREN);
1538 regs &= ~(1 << slot->id);
1539 mci_writel(slot->host, PWREN, regs);
1540 break;
1541 default:
1542 break;
1543 }
1544
1545 if (slot->host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)
1546 slot->host->state = STATE_IDLE;
1547 }
1548
1549 static int dw_mci_card_busy(struct mmc_host *mmc)
1550 {
1551 struct dw_mci_slot *slot = mmc_priv(mmc);
1552 u32 status;
1553
1554 /*
1555 * Check the busy bit which is low when DAT[3:0]
1556 * (the data lines) are 0000
1557 */
1558 status = mci_readl(slot->host, STATUS);
1559
1560 return !!(status & SDMMC_STATUS_BUSY);
1561 }
1562
1563 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
1564 {
1565 struct dw_mci_slot *slot = mmc_priv(mmc);
1566 struct dw_mci *host = slot->host;
1567 const struct dw_mci_drv_data *drv_data = host->drv_data;
1568 u32 uhs;
1569 u32 v18 = SDMMC_UHS_18V << slot->id;
1570 int ret;
1571
1572 if (drv_data && drv_data->switch_voltage)
1573 return drv_data->switch_voltage(mmc, ios);
1574
1575 /*
1576 * Program the voltage. Note that some instances of dw_mmc may use
1577 * the UHS_REG for this. For other instances (like exynos) the UHS_REG
1578 * does no harm but you need to set the regulator directly. Try both.
1579 */
1580 uhs = mci_readl(host, UHS_REG);
1581 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
1582 uhs &= ~v18;
1583 else
1584 uhs |= v18;
1585
1586 if (!IS_ERR(mmc->supply.vqmmc)) {
1587 ret = mmc_regulator_set_vqmmc(mmc, ios);
1588
1589 if (ret) {
1590 dev_dbg(&mmc->class_dev,
1591 "Regulator set error %d - %s V\n",
1592 ret, uhs & v18 ? "1.8" : "3.3");
1593 return ret;
1594 }
1595 }
1596 mci_writel(host, UHS_REG, uhs);
1597
1598 return 0;
1599 }
1600
1601 static int dw_mci_get_ro(struct mmc_host *mmc)
1602 {
1603 int read_only;
1604 struct dw_mci_slot *slot = mmc_priv(mmc);
1605 int gpio_ro = mmc_gpio_get_ro(mmc);
1606
1607 /* Use platform get_ro function, else try on board write protect */
1608 if (gpio_ro >= 0)
1609 read_only = gpio_ro;
1610 else
1611 read_only =
1612 mci_readl(slot->host, WRTPRT) & (1 << slot->id) ? 1 : 0;
1613
1614 dev_dbg(&mmc->class_dev, "card is %s\n",
1615 read_only ? "read-only" : "read-write");
1616
1617 return read_only;
1618 }
1619
1620 static void dw_mci_hw_reset(struct mmc_host *mmc)
1621 {
1622 struct dw_mci_slot *slot = mmc_priv(mmc);
1623 struct dw_mci *host = slot->host;
1624 int reset;
1625
1626 if (host->use_dma == TRANS_MODE_IDMAC)
1627 dw_mci_idmac_reset(host);
1628
1629 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET |
1630 SDMMC_CTRL_FIFO_RESET))
1631 return;
1632
1633 /*
1634 * According to eMMC spec, card reset procedure:
1635 * tRstW >= 1us: RST_n pulse width
1636 * tRSCA >= 200us: RST_n to Command time
1637 * tRSTH >= 1us: RST_n high period
1638 */
1639 reset = mci_readl(host, RST_N);
1640 reset &= ~(SDMMC_RST_HWACTIVE << slot->id);
1641 mci_writel(host, RST_N, reset);
1642 usleep_range(1, 2);
1643 reset |= SDMMC_RST_HWACTIVE << slot->id;
1644 mci_writel(host, RST_N, reset);
1645 usleep_range(200, 300);
1646 }
1647
1648 static void dw_mci_init_card(struct mmc_host *mmc, struct mmc_card *card)
1649 {
1650 struct dw_mci_slot *slot = mmc_priv(mmc);
1651 struct dw_mci *host = slot->host;
1652
1653 /*
1654 * Low power mode will stop the card clock when idle. According to the
1655 * description of the CLKENA register we should disable low power mode
1656 * for SDIO cards if we need SDIO interrupts to work.
1657 */
1658 if (mmc->caps & MMC_CAP_SDIO_IRQ) {
1659 const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR << slot->id;
1660 u32 clk_en_a_old;
1661 u32 clk_en_a;
1662
1663 clk_en_a_old = mci_readl(host, CLKENA);
1664
1665 if (card->type == MMC_TYPE_SDIO ||
1666 card->type == MMC_TYPE_SD_COMBO) {
1667 set_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
1668 clk_en_a = clk_en_a_old & ~clken_low_pwr;
1669 } else {
1670 clear_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
1671 clk_en_a = clk_en_a_old | clken_low_pwr;
1672 }
1673
1674 if (clk_en_a != clk_en_a_old) {
1675 mci_writel(host, CLKENA, clk_en_a);
1676 mci_send_cmd(slot, SDMMC_CMD_UPD_CLK |
1677 SDMMC_CMD_PRV_DAT_WAIT, 0);
1678 }
1679 }
1680 }
1681
1682 static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
1683 {
1684 struct dw_mci *host = slot->host;
1685 unsigned long irqflags;
1686 u32 int_mask;
1687
1688 spin_lock_irqsave(&host->irq_lock, irqflags);
1689
1690 /* Enable/disable Slot Specific SDIO interrupt */
1691 int_mask = mci_readl(host, INTMASK);
1692 if (enb)
1693 int_mask |= SDMMC_INT_SDIO(slot->sdio_id);
1694 else
1695 int_mask &= ~SDMMC_INT_SDIO(slot->sdio_id);
1696 mci_writel(host, INTMASK, int_mask);
1697
1698 spin_unlock_irqrestore(&host->irq_lock, irqflags);
1699 }
1700
1701 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)
1702 {
1703 struct dw_mci_slot *slot = mmc_priv(mmc);
1704 struct dw_mci *host = slot->host;
1705
1706 __dw_mci_enable_sdio_irq(slot, enb);
1707
1708 /* Avoid runtime suspending the device when SDIO IRQ is enabled */
1709 if (enb)
1710 pm_runtime_get_noresume(host->dev);
1711 else
1712 pm_runtime_put_noidle(host->dev);
1713 }
1714
1715 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc)
1716 {
1717 struct dw_mci_slot *slot = mmc_priv(mmc);
1718
1719 __dw_mci_enable_sdio_irq(slot, 1);
1720 }
1721
1722 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode)
1723 {
1724 struct dw_mci_slot *slot = mmc_priv(mmc);
1725 struct dw_mci *host = slot->host;
1726 const struct dw_mci_drv_data *drv_data = host->drv_data;
1727 int err = -EINVAL;
1728
1729 if (drv_data && drv_data->execute_tuning)
1730 err = drv_data->execute_tuning(slot, opcode);
1731 return err;
1732 }
1733
1734 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc,
1735 struct mmc_ios *ios)
1736 {
1737 struct dw_mci_slot *slot = mmc_priv(mmc);
1738 struct dw_mci *host = slot->host;
1739 const struct dw_mci_drv_data *drv_data = host->drv_data;
1740
1741 if (drv_data && drv_data->prepare_hs400_tuning)
1742 return drv_data->prepare_hs400_tuning(host, ios);
1743
1744 return 0;
1745 }
1746
1747 static bool dw_mci_reset(struct dw_mci *host)
1748 {
1749 u32 flags = SDMMC_CTRL_RESET | SDMMC_CTRL_FIFO_RESET;
1750 bool ret = false;
1751 u32 status = 0;
1752
1753 /*
1754 * Resetting generates a block interrupt, hence setting
1755 * the scatter-gather pointer to NULL.
1756 */
1757 if (host->sg) {
1758 sg_miter_stop(&host->sg_miter);
1759 host->sg = NULL;
1760 }
1761
1762 if (host->use_dma)
1763 flags |= SDMMC_CTRL_DMA_RESET;
1764
1765 if (dw_mci_ctrl_reset(host, flags)) {
1766 /*
1767 * In all cases we clear the RAWINTS
1768 * register to clear any interrupts.
1769 */
1770 mci_writel(host, RINTSTS, 0xFFFFFFFF);
1771
1772 if (!host->use_dma) {
1773 ret = true;
1774 goto ciu_out;
1775 }
1776
1777 /* Wait for dma_req to be cleared */
1778 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS,
1779 status,
1780 !(status & SDMMC_STATUS_DMA_REQ),
1781 1, 500 * USEC_PER_MSEC)) {
1782 dev_err(host->dev,
1783 "%s: Timeout waiting for dma_req to be cleared\n",
1784 __func__);
1785 goto ciu_out;
1786 }
1787
1788 /* when using DMA next we reset the fifo again */
1789 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET))
1790 goto ciu_out;
1791 } else {
1792 /* if the controller reset bit did clear, then set clock regs */
1793 if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) {
1794 dev_err(host->dev,
1795 "%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n",
1796 __func__);
1797 goto ciu_out;
1798 }
1799 }
1800
1801 if (host->use_dma == TRANS_MODE_IDMAC)
1802 /* It is also recommended that we reset and reprogram idmac */
1803 dw_mci_idmac_reset(host);
1804
1805 ret = true;
1806
1807 ciu_out:
1808 /* After a CTRL reset we need to have CIU set clock registers */
1809 mci_send_cmd(host->slot, SDMMC_CMD_UPD_CLK, 0);
1810
1811 return ret;
1812 }
1813
1814 static const struct mmc_host_ops dw_mci_ops = {
1815 .request = dw_mci_request,
1816 .pre_req = dw_mci_pre_req,
1817 .post_req = dw_mci_post_req,
1818 .set_ios = dw_mci_set_ios,
1819 .get_ro = dw_mci_get_ro,
1820 .get_cd = dw_mci_get_cd,
1821 .hw_reset = dw_mci_hw_reset,
1822 .enable_sdio_irq = dw_mci_enable_sdio_irq,
1823 .ack_sdio_irq = dw_mci_ack_sdio_irq,
1824 .execute_tuning = dw_mci_execute_tuning,
1825 .card_busy = dw_mci_card_busy,
1826 .start_signal_voltage_switch = dw_mci_switch_voltage,
1827 .init_card = dw_mci_init_card,
1828 .prepare_hs400_tuning = dw_mci_prepare_hs400_tuning,
1829 };
1830
1831 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
1832 __releases(&host->lock)
1833 __acquires(&host->lock)
1834 {
1835 struct dw_mci_slot *slot;
1836 struct mmc_host *prev_mmc = host->slot->mmc;
1837
1838 WARN_ON(host->cmd || host->data);
1839
1840 host->slot->mrq = NULL;
1841 host->mrq = NULL;
1842 if (!list_empty(&host->queue)) {
1843 slot = list_entry(host->queue.next,
1844 struct dw_mci_slot, queue_node);
1845 list_del(&slot->queue_node);
1846 dev_vdbg(host->dev, "list not empty: %s is next\n",
1847 mmc_hostname(slot->mmc));
1848 host->state = STATE_SENDING_CMD;
1849 dw_mci_start_request(host, slot);
1850 } else {
1851 dev_vdbg(host->dev, "list empty\n");
1852
1853 if (host->state == STATE_SENDING_CMD11)
1854 host->state = STATE_WAITING_CMD11_DONE;
1855 else
1856 host->state = STATE_IDLE;
1857 }
1858
1859 spin_unlock(&host->lock);
1860 mmc_request_done(prev_mmc, mrq);
1861 spin_lock(&host->lock);
1862 }
1863
1864 static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd)
1865 {
1866 u32 status = host->cmd_status;
1867
1868 host->cmd_status = 0;
1869
1870 /* Read the response from the card (up to 16 bytes) */
1871 if (cmd->flags & MMC_RSP_PRESENT) {
1872 if (cmd->flags & MMC_RSP_136) {
1873 cmd->resp[3] = mci_readl(host, RESP0);
1874 cmd->resp[2] = mci_readl(host, RESP1);
1875 cmd->resp[1] = mci_readl(host, RESP2);
1876 cmd->resp[0] = mci_readl(host, RESP3);
1877 } else {
1878 cmd->resp[0] = mci_readl(host, RESP0);
1879 cmd->resp[1] = 0;
1880 cmd->resp[2] = 0;
1881 cmd->resp[3] = 0;
1882 }
1883 }
1884
1885 if (status & SDMMC_INT_RTO)
1886 cmd->error = -ETIMEDOUT;
1887 else if ((cmd->flags & MMC_RSP_CRC) && (status & SDMMC_INT_RCRC))
1888 cmd->error = -EILSEQ;
1889 else if (status & SDMMC_INT_RESP_ERR)
1890 cmd->error = -EIO;
1891 else
1892 cmd->error = 0;
1893
1894 return cmd->error;
1895 }
1896
1897 static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)
1898 {
1899 u32 status = host->data_status;
1900
1901 if (status & DW_MCI_DATA_ERROR_FLAGS) {
1902 if (status & SDMMC_INT_DRTO) {
1903 data->error = -ETIMEDOUT;
1904 } else if (status & SDMMC_INT_DCRC) {
1905 data->error = -EILSEQ;
1906 } else if (status & SDMMC_INT_EBE) {
1907 if (host->dir_status ==
1908 DW_MCI_SEND_STATUS) {
1909 /*
1910 * No data CRC status was returned.
1911 * The number of bytes transferred
1912 * will be exaggerated in PIO mode.
1913 */
1914 data->bytes_xfered = 0;
1915 data->error = -ETIMEDOUT;
1916 } else if (host->dir_status ==
1917 DW_MCI_RECV_STATUS) {
1918 data->error = -EILSEQ;
1919 }
1920 } else {
1921 /* SDMMC_INT_SBE is included */
1922 data->error = -EILSEQ;
1923 }
1924
1925 dev_dbg(host->dev, "data error, status 0x%08x\n", status);
1926
1927 /*
1928 * After an error, there may be data lingering
1929 * in the FIFO
1930 */
1931 dw_mci_reset(host);
1932 } else {
1933 data->bytes_xfered = data->blocks * data->blksz;
1934 data->error = 0;
1935 }
1936
1937 return data->error;
1938 }
1939
1940 static void dw_mci_set_drto(struct dw_mci *host)
1941 {
1942 unsigned int drto_clks;
1943 unsigned int drto_div;
1944 unsigned int drto_ms;
1945 unsigned long irqflags;
1946
1947 drto_clks = mci_readl(host, TMOUT) >> 8;
1948 drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;
1949 if (drto_div == 0)
1950 drto_div = 1;
1951 drto_ms = DIV_ROUND_UP(MSEC_PER_SEC * drto_clks * drto_div,
1952 host->bus_hz);
1953
1954 /* add a bit spare time */
1955 drto_ms += 10;
1956
1957 spin_lock_irqsave(&host->irq_lock, irqflags);
1958 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))
1959 mod_timer(&host->dto_timer,
1960 jiffies + msecs_to_jiffies(drto_ms));
1961 spin_unlock_irqrestore(&host->irq_lock, irqflags);
1962 }
1963
1964 static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host)
1965 {
1966 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
1967 return false;
1968
1969 /*
1970 * Really be certain that the timer has stopped. This is a bit of
1971 * paranoia and could only really happen if we had really bad
1972 * interrupt latency and the interrupt routine and timeout were
1973 * running concurrently so that the del_timer() in the interrupt
1974 * handler couldn't run.
1975 */
1976 WARN_ON(del_timer_sync(&host->cto_timer));
1977 clear_bit(EVENT_CMD_COMPLETE, &host->pending_events);
1978
1979 return true;
1980 }
1981
1982 static bool dw_mci_clear_pending_data_complete(struct dw_mci *host)
1983 {
1984 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))
1985 return false;
1986
1987 /* Extra paranoia just like dw_mci_clear_pending_cmd_complete() */
1988 WARN_ON(del_timer_sync(&host->dto_timer));
1989 clear_bit(EVENT_DATA_COMPLETE, &host->pending_events);
1990
1991 return true;
1992 }
1993
1994 static void dw_mci_tasklet_func(unsigned long priv)
1995 {
1996 struct dw_mci *host = (struct dw_mci *)priv;
1997 struct mmc_data *data;
1998 struct mmc_command *cmd;
1999 struct mmc_request *mrq;
2000 enum dw_mci_state state;
2001 enum dw_mci_state prev_state;
2002 unsigned int err;
2003
2004 spin_lock(&host->lock);
2005
2006 state = host->state;
2007 data = host->data;
2008 mrq = host->mrq;
2009
2010 do {
2011 prev_state = state;
2012
2013 switch (state) {
2014 case STATE_IDLE:
2015 case STATE_WAITING_CMD11_DONE:
2016 break;
2017
2018 case STATE_SENDING_CMD11:
2019 case STATE_SENDING_CMD:
2020 if (!dw_mci_clear_pending_cmd_complete(host))
2021 break;
2022
2023 cmd = host->cmd;
2024 host->cmd = NULL;
2025 set_bit(EVENT_CMD_COMPLETE, &host->completed_events);
2026 err = dw_mci_command_complete(host, cmd);
2027 if (cmd == mrq->sbc && !err) {
2028 prev_state = state = STATE_SENDING_CMD;
2029 __dw_mci_start_request(host, host->slot,
2030 mrq->cmd);
2031 goto unlock;
2032 }
2033
2034 if (cmd->data && err) {
2035 /*
2036 * During UHS tuning sequence, sending the stop
2037 * command after the response CRC error would
2038 * throw the system into a confused state
2039 * causing all future tuning phases to report
2040 * failure.
2041 *
2042 * In such case controller will move into a data
2043 * transfer state after a response error or
2044 * response CRC error. Let's let that finish
2045 * before trying to send a stop, so we'll go to
2046 * STATE_SENDING_DATA.
2047 *
2048 * Although letting the data transfer take place
2049 * will waste a bit of time (we already know
2050 * the command was bad), it can't cause any
2051 * errors since it's possible it would have
2052 * taken place anyway if this tasklet got
2053 * delayed. Allowing the transfer to take place
2054 * avoids races and keeps things simple.
2055 */
2056 if ((err != -ETIMEDOUT) &&
2057 (cmd->opcode == MMC_SEND_TUNING_BLOCK)) {
2058 state = STATE_SENDING_DATA;
2059 continue;
2060 }
2061
2062 dw_mci_stop_dma(host);
2063 send_stop_abort(host, data);
2064 state = STATE_SENDING_STOP;
2065 break;
2066 }
2067
2068 if (!cmd->data || err) {
2069 dw_mci_request_end(host, mrq);
2070 goto unlock;
2071 }
2072
2073 prev_state = state = STATE_SENDING_DATA;
2074 /* fall through */
2075
2076 case STATE_SENDING_DATA:
2077 /*
2078 * We could get a data error and never a transfer
2079 * complete so we'd better check for it here.
2080 *
2081 * Note that we don't really care if we also got a
2082 * transfer complete; stopping the DMA and sending an
2083 * abort won't hurt.
2084 */
2085 if (test_and_clear_bit(EVENT_DATA_ERROR,
2086 &host->pending_events)) {
2087 dw_mci_stop_dma(host);
2088 if (!(host->data_status & (SDMMC_INT_DRTO |
2089 SDMMC_INT_EBE)))
2090 send_stop_abort(host, data);
2091 state = STATE_DATA_ERROR;
2092 break;
2093 }
2094
2095 if (!test_and_clear_bit(EVENT_XFER_COMPLETE,
2096 &host->pending_events)) {
2097 /*
2098 * If all data-related interrupts don't come
2099 * within the given time in reading data state.
2100 */
2101 if (host->dir_status == DW_MCI_RECV_STATUS)
2102 dw_mci_set_drto(host);
2103 break;
2104 }
2105
2106 set_bit(EVENT_XFER_COMPLETE, &host->completed_events);
2107
2108 /*
2109 * Handle an EVENT_DATA_ERROR that might have shown up
2110 * before the transfer completed. This might not have
2111 * been caught by the check above because the interrupt
2112 * could have gone off between the previous check and
2113 * the check for transfer complete.
2114 *
2115 * Technically this ought not be needed assuming we
2116 * get a DATA_COMPLETE eventually (we'll notice the
2117 * error and end the request), but it shouldn't hurt.
2118 *
2119 * This has the advantage of sending the stop command.
2120 */
2121 if (test_and_clear_bit(EVENT_DATA_ERROR,
2122 &host->pending_events)) {
2123 dw_mci_stop_dma(host);
2124 if (!(host->data_status & (SDMMC_INT_DRTO |
2125 SDMMC_INT_EBE)))
2126 send_stop_abort(host, data);
2127 state = STATE_DATA_ERROR;
2128 break;
2129 }
2130 prev_state = state = STATE_DATA_BUSY;
2131
2132 /* fall through */
2133
2134 case STATE_DATA_BUSY:
2135 if (!dw_mci_clear_pending_data_complete(host)) {
2136 /*
2137 * If data error interrupt comes but data over
2138 * interrupt doesn't come within the given time.
2139 * in reading data state.
2140 */
2141 if (host->dir_status == DW_MCI_RECV_STATUS)
2142 dw_mci_set_drto(host);
2143 break;
2144 }
2145
2146 host->data = NULL;
2147 set_bit(EVENT_DATA_COMPLETE, &host->completed_events);
2148 err = dw_mci_data_complete(host, data);
2149
2150 if (!err) {
2151 if (!data->stop || mrq->sbc) {
2152 if (mrq->sbc && data->stop)
2153 data->stop->error = 0;
2154 dw_mci_request_end(host, mrq);
2155 goto unlock;
2156 }
2157
2158 /* stop command for open-ended transfer*/
2159 if (data->stop)
2160 send_stop_abort(host, data);
2161 } else {
2162 /*
2163 * If we don't have a command complete now we'll
2164 * never get one since we just reset everything;
2165 * better end the request.
2166 *
2167 * If we do have a command complete we'll fall
2168 * through to the SENDING_STOP command and
2169 * everything will be peachy keen.
2170 */
2171 if (!test_bit(EVENT_CMD_COMPLETE,
2172 &host->pending_events)) {
2173 host->cmd = NULL;
2174 dw_mci_request_end(host, mrq);
2175 goto unlock;
2176 }
2177 }
2178
2179 /*
2180 * If err has non-zero,
2181 * stop-abort command has been already issued.
2182 */
2183 prev_state = state = STATE_SENDING_STOP;
2184
2185 /* fall through */
2186
2187 case STATE_SENDING_STOP:
2188 if (!dw_mci_clear_pending_cmd_complete(host))
2189 break;
2190
2191 /* CMD error in data command */
2192 if (mrq->cmd->error && mrq->data)
2193 dw_mci_reset(host);
2194
2195 host->cmd = NULL;
2196 host->data = NULL;
2197
2198 if (!mrq->sbc && mrq->stop)
2199 dw_mci_command_complete(host, mrq->stop);
2200 else
2201 host->cmd_status = 0;
2202
2203 dw_mci_request_end(host, mrq);
2204 goto unlock;
2205
2206 case STATE_DATA_ERROR:
2207 if (!test_and_clear_bit(EVENT_XFER_COMPLETE,
2208 &host->pending_events))
2209 break;
2210
2211 state = STATE_DATA_BUSY;
2212 break;
2213 }
2214 } while (state != prev_state);
2215
2216 host->state = state;
2217 unlock:
2218 spin_unlock(&host->lock);
2219
2220 }
2221
2222 /* push final bytes to part_buf, only use during push */
2223 static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt)
2224 {
2225 memcpy((void *)&host->part_buf, buf, cnt);
2226 host->part_buf_count = cnt;
2227 }
2228
2229 /* append bytes to part_buf, only use during push */
2230 static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt)
2231 {
2232 cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count);
2233 memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt);
2234 host->part_buf_count += cnt;
2235 return cnt;
2236 }
2237
2238 /* pull first bytes from part_buf, only use during pull */
2239 static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt)
2240 {
2241 cnt = min_t(int, cnt, host->part_buf_count);
2242 if (cnt) {
2243 memcpy(buf, (void *)&host->part_buf + host->part_buf_start,
2244 cnt);
2245 host->part_buf_count -= cnt;
2246 host->part_buf_start += cnt;
2247 }
2248 return cnt;
2249 }
2250
2251 /* pull final bytes from the part_buf, assuming it's just been filled */
2252 static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt)
2253 {
2254 memcpy(buf, &host->part_buf, cnt);
2255 host->part_buf_start = cnt;
2256 host->part_buf_count = (1 << host->data_shift) - cnt;
2257 }
2258
2259 static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt)
2260 {
2261 struct mmc_data *data = host->data;
2262 int init_cnt = cnt;
2263
2264 /* try and push anything in the part_buf */
2265 if (unlikely(host->part_buf_count)) {
2266 int len = dw_mci_push_part_bytes(host, buf, cnt);
2267
2268 buf += len;
2269 cnt -= len;
2270 if (host->part_buf_count == 2) {
2271 mci_fifo_writew(host->fifo_reg, host->part_buf16);
2272 host->part_buf_count = 0;
2273 }
2274 }
2275 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2276 if (unlikely((unsigned long)buf & 0x1)) {
2277 while (cnt >= 2) {
2278 u16 aligned_buf[64];
2279 int len = min(cnt & -2, (int)sizeof(aligned_buf));
2280 int items = len >> 1;
2281 int i;
2282 /* memcpy from input buffer into aligned buffer */
2283 memcpy(aligned_buf, buf, len);
2284 buf += len;
2285 cnt -= len;
2286 /* push data from aligned buffer into fifo */
2287 for (i = 0; i < items; ++i)
2288 mci_fifo_writew(host->fifo_reg, aligned_buf[i]);
2289 }
2290 } else
2291 #endif
2292 {
2293 u16 *pdata = buf;
2294
2295 for (; cnt >= 2; cnt -= 2)
2296 mci_fifo_writew(host->fifo_reg, *pdata++);
2297 buf = pdata;
2298 }
2299 /* put anything remaining in the part_buf */
2300 if (cnt) {
2301 dw_mci_set_part_bytes(host, buf, cnt);
2302 /* Push data if we have reached the expected data length */
2303 if ((data->bytes_xfered + init_cnt) ==
2304 (data->blksz * data->blocks))
2305 mci_fifo_writew(host->fifo_reg, host->part_buf16);
2306 }
2307 }
2308
2309 static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt)
2310 {
2311 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2312 if (unlikely((unsigned long)buf & 0x1)) {
2313 while (cnt >= 2) {
2314 /* pull data from fifo into aligned buffer */
2315 u16 aligned_buf[64];
2316 int len = min(cnt & -2, (int)sizeof(aligned_buf));
2317 int items = len >> 1;
2318 int i;
2319
2320 for (i = 0; i < items; ++i)
2321 aligned_buf[i] = mci_fifo_readw(host->fifo_reg);
2322 /* memcpy from aligned buffer into output buffer */
2323 memcpy(buf, aligned_buf, len);
2324 buf += len;
2325 cnt -= len;
2326 }
2327 } else
2328 #endif
2329 {
2330 u16 *pdata = buf;
2331
2332 for (; cnt >= 2; cnt -= 2)
2333 *pdata++ = mci_fifo_readw(host->fifo_reg);
2334 buf = pdata;
2335 }
2336 if (cnt) {
2337 host->part_buf16 = mci_fifo_readw(host->fifo_reg);
2338 dw_mci_pull_final_bytes(host, buf, cnt);
2339 }
2340 }
2341
2342 static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt)
2343 {
2344 struct mmc_data *data = host->data;
2345 int init_cnt = cnt;
2346
2347 /* try and push anything in the part_buf */
2348 if (unlikely(host->part_buf_count)) {
2349 int len = dw_mci_push_part_bytes(host, buf, cnt);
2350
2351 buf += len;
2352 cnt -= len;
2353 if (host->part_buf_count == 4) {
2354 mci_fifo_writel(host->fifo_reg, host->part_buf32);
2355 host->part_buf_count = 0;
2356 }
2357 }
2358 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2359 if (unlikely((unsigned long)buf & 0x3)) {
2360 while (cnt >= 4) {
2361 u32 aligned_buf[32];
2362 int len = min(cnt & -4, (int)sizeof(aligned_buf));
2363 int items = len >> 2;
2364 int i;
2365 /* memcpy from input buffer into aligned buffer */
2366 memcpy(aligned_buf, buf, len);
2367 buf += len;
2368 cnt -= len;
2369 /* push data from aligned buffer into fifo */
2370 for (i = 0; i < items; ++i)
2371 mci_fifo_writel(host->fifo_reg, aligned_buf[i]);
2372 }
2373 } else
2374 #endif
2375 {
2376 u32 *pdata = buf;
2377
2378 for (; cnt >= 4; cnt -= 4)
2379 mci_fifo_writel(host->fifo_reg, *pdata++);
2380 buf = pdata;
2381 }
2382 /* put anything remaining in the part_buf */
2383 if (cnt) {
2384 dw_mci_set_part_bytes(host, buf, cnt);
2385 /* Push data if we have reached the expected data length */
2386 if ((data->bytes_xfered + init_cnt) ==
2387 (data->blksz * data->blocks))
2388 mci_fifo_writel(host->fifo_reg, host->part_buf32);
2389 }
2390 }
2391
2392 static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt)
2393 {
2394 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2395 if (unlikely((unsigned long)buf & 0x3)) {
2396 while (cnt >= 4) {
2397 /* pull data from fifo into aligned buffer */
2398 u32 aligned_buf[32];
2399 int len = min(cnt & -4, (int)sizeof(aligned_buf));
2400 int items = len >> 2;
2401 int i;
2402
2403 for (i = 0; i < items; ++i)
2404 aligned_buf[i] = mci_fifo_readl(host->fifo_reg);
2405 /* memcpy from aligned buffer into output buffer */
2406 memcpy(buf, aligned_buf, len);
2407 buf += len;
2408 cnt -= len;
2409 }
2410 } else
2411 #endif
2412 {
2413 u32 *pdata = buf;
2414
2415 for (; cnt >= 4; cnt -= 4)
2416 *pdata++ = mci_fifo_readl(host->fifo_reg);
2417 buf = pdata;
2418 }
2419 if (cnt) {
2420 host->part_buf32 = mci_fifo_readl(host->fifo_reg);
2421 dw_mci_pull_final_bytes(host, buf, cnt);
2422 }
2423 }
2424
2425 static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt)
2426 {
2427 struct mmc_data *data = host->data;
2428 int init_cnt = cnt;
2429
2430 /* try and push anything in the part_buf */
2431 if (unlikely(host->part_buf_count)) {
2432 int len = dw_mci_push_part_bytes(host, buf, cnt);
2433
2434 buf += len;
2435 cnt -= len;
2436
2437 if (host->part_buf_count == 8) {
2438 mci_fifo_writeq(host->fifo_reg, host->part_buf);
2439 host->part_buf_count = 0;
2440 }
2441 }
2442 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2443 if (unlikely((unsigned long)buf & 0x7)) {
2444 while (cnt >= 8) {
2445 u64 aligned_buf[16];
2446 int len = min(cnt & -8, (int)sizeof(aligned_buf));
2447 int items = len >> 3;
2448 int i;
2449 /* memcpy from input buffer into aligned buffer */
2450 memcpy(aligned_buf, buf, len);
2451 buf += len;
2452 cnt -= len;
2453 /* push data from aligned buffer into fifo */
2454 for (i = 0; i < items; ++i)
2455 mci_fifo_writeq(host->fifo_reg, aligned_buf[i]);
2456 }
2457 } else
2458 #endif
2459 {
2460 u64 *pdata = buf;
2461
2462 for (; cnt >= 8; cnt -= 8)
2463 mci_fifo_writeq(host->fifo_reg, *pdata++);
2464 buf = pdata;
2465 }
2466 /* put anything remaining in the part_buf */
2467 if (cnt) {
2468 dw_mci_set_part_bytes(host, buf, cnt);
2469 /* Push data if we have reached the expected data length */
2470 if ((data->bytes_xfered + init_cnt) ==
2471 (data->blksz * data->blocks))
2472 mci_fifo_writeq(host->fifo_reg, host->part_buf);
2473 }
2474 }
2475
2476 static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt)
2477 {
2478 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2479 if (unlikely((unsigned long)buf & 0x7)) {
2480 while (cnt >= 8) {
2481 /* pull data from fifo into aligned buffer */
2482 u64 aligned_buf[16];
2483 int len = min(cnt & -8, (int)sizeof(aligned_buf));
2484 int items = len >> 3;
2485 int i;
2486
2487 for (i = 0; i < items; ++i)
2488 aligned_buf[i] = mci_fifo_readq(host->fifo_reg);
2489
2490 /* memcpy from aligned buffer into output buffer */
2491 memcpy(buf, aligned_buf, len);
2492 buf += len;
2493 cnt -= len;
2494 }
2495 } else
2496 #endif
2497 {
2498 u64 *pdata = buf;
2499
2500 for (; cnt >= 8; cnt -= 8)
2501 *pdata++ = mci_fifo_readq(host->fifo_reg);
2502 buf = pdata;
2503 }
2504 if (cnt) {
2505 host->part_buf = mci_fifo_readq(host->fifo_reg);
2506 dw_mci_pull_final_bytes(host, buf, cnt);
2507 }
2508 }
2509
2510 static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt)
2511 {
2512 int len;
2513
2514 /* get remaining partial bytes */
2515 len = dw_mci_pull_part_bytes(host, buf, cnt);
2516 if (unlikely(len == cnt))
2517 return;
2518 buf += len;
2519 cnt -= len;
2520
2521 /* get the rest of the data */
2522 host->pull_data(host, buf, cnt);
2523 }
2524
2525 static void dw_mci_read_data_pio(struct dw_mci *host, bool dto)
2526 {
2527 struct sg_mapping_iter *sg_miter = &host->sg_miter;
2528 void *buf;
2529 unsigned int offset;
2530 struct mmc_data *data = host->data;
2531 int shift = host->data_shift;
2532 u32 status;
2533 unsigned int len;
2534 unsigned int remain, fcnt;
2535
2536 do {
2537 if (!sg_miter_next(sg_miter))
2538 goto done;
2539
2540 host->sg = sg_miter->piter.sg;
2541 buf = sg_miter->addr;
2542 remain = sg_miter->length;
2543 offset = 0;
2544
2545 do {
2546 fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS))
2547 << shift) + host->part_buf_count;
2548 len = min(remain, fcnt);
2549 if (!len)
2550 break;
2551 dw_mci_pull_data(host, (void *)(buf + offset), len);
2552 data->bytes_xfered += len;
2553 offset += len;
2554 remain -= len;
2555 } while (remain);
2556
2557 sg_miter->consumed = offset;
2558 status = mci_readl(host, MINTSTS);
2559 mci_writel(host, RINTSTS, SDMMC_INT_RXDR);
2560 /* if the RXDR is ready read again */
2561 } while ((status & SDMMC_INT_RXDR) ||
2562 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS))));
2563
2564 if (!remain) {
2565 if (!sg_miter_next(sg_miter))
2566 goto done;
2567 sg_miter->consumed = 0;
2568 }
2569 sg_miter_stop(sg_miter);
2570 return;
2571
2572 done:
2573 sg_miter_stop(sg_miter);
2574 host->sg = NULL;
2575 smp_wmb(); /* drain writebuffer */
2576 set_bit(EVENT_XFER_COMPLETE, &host->pending_events);
2577 }
2578
2579 static void dw_mci_write_data_pio(struct dw_mci *host)
2580 {
2581 struct sg_mapping_iter *sg_miter = &host->sg_miter;
2582 void *buf;
2583 unsigned int offset;
2584 struct mmc_data *data = host->data;
2585 int shift = host->data_shift;
2586 u32 status;
2587 unsigned int len;
2588 unsigned int fifo_depth = host->fifo_depth;
2589 unsigned int remain, fcnt;
2590
2591 do {
2592 if (!sg_miter_next(sg_miter))
2593 goto done;
2594
2595 host->sg = sg_miter->piter.sg;
2596 buf = sg_miter->addr;
2597 remain = sg_miter->length;
2598 offset = 0;
2599
2600 do {
2601 fcnt = ((fifo_depth -
2602 SDMMC_GET_FCNT(mci_readl(host, STATUS)))
2603 << shift) - host->part_buf_count;
2604 len = min(remain, fcnt);
2605 if (!len)
2606 break;
2607 host->push_data(host, (void *)(buf + offset), len);
2608 data->bytes_xfered += len;
2609 offset += len;
2610 remain -= len;
2611 } while (remain);
2612
2613 sg_miter->consumed = offset;
2614 status = mci_readl(host, MINTSTS);
2615 mci_writel(host, RINTSTS, SDMMC_INT_TXDR);
2616 } while (status & SDMMC_INT_TXDR); /* if TXDR write again */
2617
2618 if (!remain) {
2619 if (!sg_miter_next(sg_miter))
2620 goto done;
2621 sg_miter->consumed = 0;
2622 }
2623 sg_miter_stop(sg_miter);
2624 return;
2625
2626 done:
2627 sg_miter_stop(sg_miter);
2628 host->sg = NULL;
2629 smp_wmb(); /* drain writebuffer */
2630 set_bit(EVENT_XFER_COMPLETE, &host->pending_events);
2631 }
2632
2633 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
2634 {
2635 del_timer(&host->cto_timer);
2636
2637 if (!host->cmd_status)
2638 host->cmd_status = status;
2639
2640 smp_wmb(); /* drain writebuffer */
2641
2642 set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
2643 tasklet_schedule(&host->tasklet);
2644 }
2645
2646 static void dw_mci_handle_cd(struct dw_mci *host)
2647 {
2648 struct dw_mci_slot *slot = host->slot;
2649
2650 if (slot->mmc->ops->card_event)
2651 slot->mmc->ops->card_event(slot->mmc);
2652 mmc_detect_change(slot->mmc,
2653 msecs_to_jiffies(host->pdata->detect_delay_ms));
2654 }
2655
2656 static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
2657 {
2658 struct dw_mci *host = dev_id;
2659 u32 pending;
2660 struct dw_mci_slot *slot = host->slot;
2661 unsigned long irqflags;
2662
2663 pending = mci_readl(host, MINTSTS); /* read-only mask reg */
2664
2665 if (pending) {
2666 /* Check volt switch first, since it can look like an error */
2667 if ((host->state == STATE_SENDING_CMD11) &&
2668 (pending & SDMMC_INT_VOLT_SWITCH)) {
2669 mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH);
2670 pending &= ~SDMMC_INT_VOLT_SWITCH;
2671
2672 /*
2673 * Hold the lock; we know cmd11_timer can't be kicked
2674 * off after the lock is released, so safe to delete.
2675 */
2676 spin_lock_irqsave(&host->irq_lock, irqflags);
2677 dw_mci_cmd_interrupt(host, pending);
2678 spin_unlock_irqrestore(&host->irq_lock, irqflags);
2679
2680 del_timer(&host->cmd11_timer);
2681 }
2682
2683 if (pending & DW_MCI_CMD_ERROR_FLAGS) {
2684 spin_lock_irqsave(&host->irq_lock, irqflags);
2685
2686 del_timer(&host->cto_timer);
2687 mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS);
2688 host->cmd_status = pending;
2689 smp_wmb(); /* drain writebuffer */
2690 set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
2691
2692 spin_unlock_irqrestore(&host->irq_lock, irqflags);
2693 }
2694
2695 if (pending & DW_MCI_DATA_ERROR_FLAGS) {
2696 /* if there is an error report DATA_ERROR */
2697 mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS);
2698 host->data_status = pending;
2699 smp_wmb(); /* drain writebuffer */
2700 set_bit(EVENT_DATA_ERROR, &host->pending_events);
2701 tasklet_schedule(&host->tasklet);
2702 }
2703
2704 if (pending & SDMMC_INT_DATA_OVER) {
2705 spin_lock_irqsave(&host->irq_lock, irqflags);
2706
2707 del_timer(&host->dto_timer);
2708
2709 mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER);
2710 if (!host->data_status)
2711 host->data_status = pending;
2712 smp_wmb(); /* drain writebuffer */
2713 if (host->dir_status == DW_MCI_RECV_STATUS) {
2714 if (host->sg != NULL)
2715 dw_mci_read_data_pio(host, true);
2716 }
2717 set_bit(EVENT_DATA_COMPLETE, &host->pending_events);
2718 tasklet_schedule(&host->tasklet);
2719
2720 spin_unlock_irqrestore(&host->irq_lock, irqflags);
2721 }
2722
2723 if (pending & SDMMC_INT_RXDR) {
2724 mci_writel(host, RINTSTS, SDMMC_INT_RXDR);
2725 if (host->dir_status == DW_MCI_RECV_STATUS && host->sg)
2726 dw_mci_read_data_pio(host, false);
2727 }
2728
2729 if (pending & SDMMC_INT_TXDR) {
2730 mci_writel(host, RINTSTS, SDMMC_INT_TXDR);
2731 if (host->dir_status == DW_MCI_SEND_STATUS && host->sg)
2732 dw_mci_write_data_pio(host);
2733 }
2734
2735 if (pending & SDMMC_INT_CMD_DONE) {
2736 spin_lock_irqsave(&host->irq_lock, irqflags);
2737
2738 mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE);
2739 dw_mci_cmd_interrupt(host, pending);
2740
2741 spin_unlock_irqrestore(&host->irq_lock, irqflags);
2742 }
2743
2744 if (pending & SDMMC_INT_CD) {
2745 mci_writel(host, RINTSTS, SDMMC_INT_CD);
2746 dw_mci_handle_cd(host);
2747 }
2748
2749 if (pending & SDMMC_INT_SDIO(slot->sdio_id)) {
2750 mci_writel(host, RINTSTS,
2751 SDMMC_INT_SDIO(slot->sdio_id));
2752 __dw_mci_enable_sdio_irq(slot, 0);
2753 sdio_signal_irq(slot->mmc);
2754 }
2755
2756 }
2757
2758 if (host->use_dma != TRANS_MODE_IDMAC)
2759 return IRQ_HANDLED;
2760
2761 /* Handle IDMA interrupts */
2762 if (host->dma_64bit_address == 1) {
2763 pending = mci_readl(host, IDSTS64);
2764 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) {
2765 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI |
2766 SDMMC_IDMAC_INT_RI);
2767 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI);
2768 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events))
2769 host->dma_ops->complete((void *)host);
2770 }
2771 } else {
2772 pending = mci_readl(host, IDSTS);
2773 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) {
2774 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI |
2775 SDMMC_IDMAC_INT_RI);
2776 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI);
2777 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events))
2778 host->dma_ops->complete((void *)host);
2779 }
2780 }
2781
2782 return IRQ_HANDLED;
2783 }
2784
2785 static int dw_mci_init_slot(struct dw_mci *host)
2786 {
2787 struct mmc_host *mmc;
2788 struct dw_mci_slot *slot;
2789 const struct dw_mci_drv_data *drv_data = host->drv_data;
2790 int ctrl_id, ret;
2791 u32 freq[2];
2792
2793 mmc = mmc_alloc_host(sizeof(struct dw_mci_slot), host->dev);
2794 if (!mmc)
2795 return -ENOMEM;
2796
2797 slot = mmc_priv(mmc);
2798 slot->id = 0;
2799 slot->sdio_id = host->sdio_id0 + slot->id;
2800 slot->mmc = mmc;
2801 slot->host = host;
2802 host->slot = slot;
2803
2804 mmc->ops = &dw_mci_ops;
2805 if (device_property_read_u32_array(host->dev, "clock-freq-min-max",
2806 freq, 2)) {
2807 mmc->f_min = DW_MCI_FREQ_MIN;
2808 mmc->f_max = DW_MCI_FREQ_MAX;
2809 } else {
2810 dev_info(host->dev,
2811 "'clock-freq-min-max' property was deprecated.\n");
2812 mmc->f_min = freq[0];
2813 mmc->f_max = freq[1];
2814 }
2815
2816 /*if there are external regulators, get them*/
2817 ret = mmc_regulator_get_supply(mmc);
2818 if (ret)
2819 goto err_host_allocated;
2820
2821 if (!mmc->ocr_avail)
2822 mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
2823
2824 if (host->pdata->caps)
2825 mmc->caps = host->pdata->caps;
2826
2827 /*
2828 * Support MMC_CAP_ERASE by default.
2829 * It needs to use trim/discard/erase commands.
2830 */
2831 mmc->caps |= MMC_CAP_ERASE;
2832
2833 if (host->pdata->pm_caps)
2834 mmc->pm_caps = host->pdata->pm_caps;
2835
2836 if (host->dev->of_node) {
2837 ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
2838 if (ctrl_id < 0)
2839 ctrl_id = 0;
2840 } else {
2841 ctrl_id = to_platform_device(host->dev)->id;
2842 }
2843 if (drv_data && drv_data->caps)
2844 mmc->caps |= drv_data->caps[ctrl_id];
2845
2846 if (host->pdata->caps2)
2847 mmc->caps2 = host->pdata->caps2;
2848
2849 ret = mmc_of_parse(mmc);
2850 if (ret)
2851 goto err_host_allocated;
2852
2853 /* Process SDIO IRQs through the sdio_irq_work. */
2854 if (mmc->caps & MMC_CAP_SDIO_IRQ)
2855 mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
2856
2857 /* Useful defaults if platform data is unset. */
2858 if (host->use_dma == TRANS_MODE_IDMAC) {
2859 mmc->max_segs = host->ring_size;
2860 mmc->max_blk_size = 65535;
2861 mmc->max_seg_size = 0x1000;
2862 mmc->max_req_size = mmc->max_seg_size * host->ring_size;
2863 mmc->max_blk_count = mmc->max_req_size / 512;
2864 } else if (host->use_dma == TRANS_MODE_EDMAC) {
2865 mmc->max_segs = 64;
2866 mmc->max_blk_size = 65535;
2867 mmc->max_blk_count = 65535;
2868 mmc->max_req_size =
2869 mmc->max_blk_size * mmc->max_blk_count;
2870 mmc->max_seg_size = mmc->max_req_size;
2871 } else {
2872 /* TRANS_MODE_PIO */
2873 mmc->max_segs = 64;
2874 mmc->max_blk_size = 65535; /* BLKSIZ is 16 bits */
2875 mmc->max_blk_count = 512;
2876 mmc->max_req_size = mmc->max_blk_size *
2877 mmc->max_blk_count;
2878 mmc->max_seg_size = mmc->max_req_size;
2879 }
2880
2881 dw_mci_get_cd(mmc);
2882
2883 ret = mmc_add_host(mmc);
2884 if (ret)
2885 goto err_host_allocated;
2886
2887 #if defined(CONFIG_DEBUG_FS)
2888 dw_mci_init_debugfs(slot);
2889 #endif
2890
2891 return 0;
2892
2893 err_host_allocated:
2894 mmc_free_host(mmc);
2895 return ret;
2896 }
2897
2898 static void dw_mci_cleanup_slot(struct dw_mci_slot *slot)
2899 {
2900 /* Debugfs stuff is cleaned up by mmc core */
2901 mmc_remove_host(slot->mmc);
2902 slot->host->slot = NULL;
2903 mmc_free_host(slot->mmc);
2904 }
2905
2906 static void dw_mci_init_dma(struct dw_mci *host)
2907 {
2908 int addr_config;
2909 struct device *dev = host->dev;
2910
2911 /*
2912 * Check tansfer mode from HCON[17:16]
2913 * Clear the ambiguous description of dw_mmc databook:
2914 * 2b'00: No DMA Interface -> Actually means using Internal DMA block
2915 * 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block
2916 * 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block
2917 * 2b'11: Non DW DMA Interface -> pio only
2918 * Compared to DesignWare DMA Interface, Generic DMA Interface has a
2919 * simpler request/acknowledge handshake mechanism and both of them
2920 * are regarded as external dma master for dw_mmc.
2921 */
2922 host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON));
2923 if (host->use_dma == DMA_INTERFACE_IDMA) {
2924 host->use_dma = TRANS_MODE_IDMAC;
2925 } else if (host->use_dma == DMA_INTERFACE_DWDMA ||
2926 host->use_dma == DMA_INTERFACE_GDMA) {
2927 host->use_dma = TRANS_MODE_EDMAC;
2928 } else {
2929 goto no_dma;
2930 }
2931
2932 /* Determine which DMA interface to use */
2933 if (host->use_dma == TRANS_MODE_IDMAC) {
2934 /*
2935 * Check ADDR_CONFIG bit in HCON to find
2936 * IDMAC address bus width
2937 */
2938 addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON));
2939
2940 if (addr_config == 1) {
2941 /* host supports IDMAC in 64-bit address mode */
2942 host->dma_64bit_address = 1;
2943 dev_info(host->dev,
2944 "IDMAC supports 64-bit address mode.\n");
2945 if (!dma_set_mask(host->dev, DMA_BIT_MASK(64)))
2946 dma_set_coherent_mask(host->dev,
2947 DMA_BIT_MASK(64));
2948 } else {
2949 /* host supports IDMAC in 32-bit address mode */
2950 host->dma_64bit_address = 0;
2951 dev_info(host->dev,
2952 "IDMAC supports 32-bit address mode.\n");
2953 }
2954
2955 /* Alloc memory for sg translation */
2956 host->sg_cpu = dmam_alloc_coherent(host->dev,
2957 DESC_RING_BUF_SZ,
2958 &host->sg_dma, GFP_KERNEL);
2959 if (!host->sg_cpu) {
2960 dev_err(host->dev,
2961 "%s: could not alloc DMA memory\n",
2962 __func__);
2963 goto no_dma;
2964 }
2965
2966 host->dma_ops = &dw_mci_idmac_ops;
2967 dev_info(host->dev, "Using internal DMA controller.\n");
2968 } else {
2969 /* TRANS_MODE_EDMAC: check dma bindings again */
2970 if ((device_property_read_string_array(dev, "dma-names",
2971 NULL, 0) < 0) ||
2972 !device_property_present(dev, "dmas")) {
2973 goto no_dma;
2974 }
2975 host->dma_ops = &dw_mci_edmac_ops;
2976 dev_info(host->dev, "Using external DMA controller.\n");
2977 }
2978
2979 if (host->dma_ops->init && host->dma_ops->start &&
2980 host->dma_ops->stop && host->dma_ops->cleanup) {
2981 if (host->dma_ops->init(host)) {
2982 dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n",
2983 __func__);
2984 goto no_dma;
2985 }
2986 } else {
2987 dev_err(host->dev, "DMA initialization not found.\n");
2988 goto no_dma;
2989 }
2990
2991 return;
2992
2993 no_dma:
2994 dev_info(host->dev, "Using PIO mode.\n");
2995 host->use_dma = TRANS_MODE_PIO;
2996 }
2997
2998 static void dw_mci_cmd11_timer(struct timer_list *t)
2999 {
3000 struct dw_mci *host = from_timer(host, t, cmd11_timer);
3001
3002 if (host->state != STATE_SENDING_CMD11) {
3003 dev_warn(host->dev, "Unexpected CMD11 timeout\n");
3004 return;
3005 }
3006
3007 host->cmd_status = SDMMC_INT_RTO;
3008 set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
3009 tasklet_schedule(&host->tasklet);
3010 }
3011
3012 static void dw_mci_cto_timer(struct timer_list *t)
3013 {
3014 struct dw_mci *host = from_timer(host, t, cto_timer);
3015 unsigned long irqflags;
3016 u32 pending;
3017
3018 spin_lock_irqsave(&host->irq_lock, irqflags);
3019
3020 /*
3021 * If somehow we have very bad interrupt latency it's remotely possible
3022 * that the timer could fire while the interrupt is still pending or
3023 * while the interrupt is midway through running. Let's be paranoid
3024 * and detect those two cases. Note that this is paranoia is somewhat
3025 * justified because in this function we don't actually cancel the
3026 * pending command in the controller--we just assume it will never come.
3027 */
3028 pending = mci_readl(host, MINTSTS); /* read-only mask reg */
3029 if (pending & (DW_MCI_CMD_ERROR_FLAGS | SDMMC_INT_CMD_DONE)) {
3030 /* The interrupt should fire; no need to act but we can warn */
3031 dev_warn(host->dev, "Unexpected interrupt latency\n");
3032 goto exit;
3033 }
3034 if (test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) {
3035 /* Presumably interrupt handler couldn't delete the timer */
3036 dev_warn(host->dev, "CTO timeout when already completed\n");
3037 goto exit;
3038 }
3039
3040 /*
3041 * Continued paranoia to make sure we're in the state we expect.
3042 * This paranoia isn't really justified but it seems good to be safe.
3043 */
3044 switch (host->state) {
3045 case STATE_SENDING_CMD11:
3046 case STATE_SENDING_CMD:
3047 case STATE_SENDING_STOP:
3048 /*
3049 * If CMD_DONE interrupt does NOT come in sending command
3050 * state, we should notify the driver to terminate current
3051 * transfer and report a command timeout to the core.
3052 */
3053 host->cmd_status = SDMMC_INT_RTO;
3054 set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
3055 tasklet_schedule(&host->tasklet);
3056 break;
3057 default:
3058 dev_warn(host->dev, "Unexpected command timeout, state %d\n",
3059 host->state);
3060 break;
3061 }
3062
3063 exit:
3064 spin_unlock_irqrestore(&host->irq_lock, irqflags);
3065 }
3066
3067 static void dw_mci_dto_timer(struct timer_list *t)
3068 {
3069 struct dw_mci *host = from_timer(host, t, dto_timer);
3070 unsigned long irqflags;
3071 u32 pending;
3072
3073 spin_lock_irqsave(&host->irq_lock, irqflags);
3074
3075 /*
3076 * The DTO timer is much longer than the CTO timer, so it's even less
3077 * likely that we'll these cases, but it pays to be paranoid.
3078 */
3079 pending = mci_readl(host, MINTSTS); /* read-only mask reg */
3080 if (pending & SDMMC_INT_DATA_OVER) {
3081 /* The interrupt should fire; no need to act but we can warn */
3082 dev_warn(host->dev, "Unexpected data interrupt latency\n");
3083 goto exit;
3084 }
3085 if (test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) {
3086 /* Presumably interrupt handler couldn't delete the timer */
3087 dev_warn(host->dev, "DTO timeout when already completed\n");
3088 goto exit;
3089 }
3090
3091 /*
3092 * Continued paranoia to make sure we're in the state we expect.
3093 * This paranoia isn't really justified but it seems good to be safe.
3094 */
3095 switch (host->state) {
3096 case STATE_SENDING_DATA:
3097 case STATE_DATA_BUSY:
3098 /*
3099 * If DTO interrupt does NOT come in sending data state,
3100 * we should notify the driver to terminate current transfer
3101 * and report a data timeout to the core.
3102 */
3103 host->data_status = SDMMC_INT_DRTO;
3104 set_bit(EVENT_DATA_ERROR, &host->pending_events);
3105 set_bit(EVENT_DATA_COMPLETE, &host->pending_events);
3106 tasklet_schedule(&host->tasklet);
3107 break;
3108 default:
3109 dev_warn(host->dev, "Unexpected data timeout, state %d\n",
3110 host->state);
3111 break;
3112 }
3113
3114 exit:
3115 spin_unlock_irqrestore(&host->irq_lock, irqflags);
3116 }
3117
3118 #ifdef CONFIG_OF
3119 static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
3120 {
3121 struct dw_mci_board *pdata;
3122 struct device *dev = host->dev;
3123 const struct dw_mci_drv_data *drv_data = host->drv_data;
3124 int ret;
3125 u32 clock_frequency;
3126
3127 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
3128 if (!pdata)
3129 return ERR_PTR(-ENOMEM);
3130
3131 /* find reset controller when exist */
3132 pdata->rstc = devm_reset_control_get_optional_exclusive(dev, "reset");
3133 if (IS_ERR(pdata->rstc)) {
3134 if (PTR_ERR(pdata->rstc) == -EPROBE_DEFER)
3135 return ERR_PTR(-EPROBE_DEFER);
3136 }
3137
3138 /* find out number of slots supported */
3139 if (!device_property_read_u32(dev, "num-slots", &pdata->num_slots))
3140 dev_info(dev, "'num-slots' was deprecated.\n");
3141
3142 if (device_property_read_u32(dev, "fifo-depth", &pdata->fifo_depth))
3143 dev_info(dev,
3144 "fifo-depth property not found, using value of FIFOTH register as default\n");
3145
3146 device_property_read_u32(dev, "card-detect-delay",
3147 &pdata->detect_delay_ms);
3148
3149 device_property_read_u32(dev, "data-addr", &host->data_addr_override);
3150
3151 if (device_property_present(dev, "fifo-watermark-aligned"))
3152 host->wm_aligned = true;
3153
3154 if (!device_property_read_u32(dev, "clock-frequency", &clock_frequency))
3155 pdata->bus_hz = clock_frequency;
3156
3157 if (drv_data && drv_data->parse_dt) {
3158 ret = drv_data->parse_dt(host);
3159 if (ret)
3160 return ERR_PTR(ret);
3161 }
3162
3163 return pdata;
3164 }
3165
3166 #else /* CONFIG_OF */
3167 static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
3168 {
3169 return ERR_PTR(-EINVAL);
3170 }
3171 #endif /* CONFIG_OF */
3172
3173 static void dw_mci_enable_cd(struct dw_mci *host)
3174 {
3175 unsigned long irqflags;
3176 u32 temp;
3177
3178 /*
3179 * No need for CD if all slots have a non-error GPIO
3180 * as well as broken card detection is found.
3181 */
3182 if (host->slot->mmc->caps & MMC_CAP_NEEDS_POLL)
3183 return;
3184
3185 if (mmc_gpio_get_cd(host->slot->mmc) < 0) {
3186 spin_lock_irqsave(&host->irq_lock, irqflags);
3187 temp = mci_readl(host, INTMASK);
3188 temp |= SDMMC_INT_CD;
3189 mci_writel(host, INTMASK, temp);
3190 spin_unlock_irqrestore(&host->irq_lock, irqflags);
3191 }
3192 }
3193
3194 int dw_mci_probe(struct dw_mci *host)
3195 {
3196 const struct dw_mci_drv_data *drv_data = host->drv_data;
3197 int width, i, ret = 0;
3198 u32 fifo_size;
3199
3200 if (!host->pdata) {
3201 host->pdata = dw_mci_parse_dt(host);
3202 if (PTR_ERR(host->pdata) == -EPROBE_DEFER) {
3203 return -EPROBE_DEFER;
3204 } else if (IS_ERR(host->pdata)) {
3205 dev_err(host->dev, "platform data not available\n");
3206 return -EINVAL;
3207 }
3208 }
3209
3210 host->biu_clk = devm_clk_get(host->dev, "biu");
3211 if (IS_ERR(host->biu_clk)) {
3212 dev_dbg(host->dev, "biu clock not available\n");
3213 } else {
3214 ret = clk_prepare_enable(host->biu_clk);
3215 if (ret) {
3216 dev_err(host->dev, "failed to enable biu clock\n");
3217 return ret;
3218 }
3219 }
3220
3221 host->ciu_clk = devm_clk_get(host->dev, "ciu");
3222 if (IS_ERR(host->ciu_clk)) {
3223 dev_dbg(host->dev, "ciu clock not available\n");
3224 host->bus_hz = host->pdata->bus_hz;
3225 } else {
3226 ret = clk_prepare_enable(host->ciu_clk);
3227 if (ret) {
3228 dev_err(host->dev, "failed to enable ciu clock\n");
3229 goto err_clk_biu;
3230 }
3231
3232 if (host->pdata->bus_hz) {
3233 ret = clk_set_rate(host->ciu_clk, host->pdata->bus_hz);
3234 if (ret)
3235 dev_warn(host->dev,
3236 "Unable to set bus rate to %uHz\n",
3237 host->pdata->bus_hz);
3238 }
3239 host->bus_hz = clk_get_rate(host->ciu_clk);
3240 }
3241
3242 if (!host->bus_hz) {
3243 dev_err(host->dev,
3244 "Platform data must supply bus speed\n");
3245 ret = -ENODEV;
3246 goto err_clk_ciu;
3247 }
3248
3249 if (!IS_ERR(host->pdata->rstc)) {
3250 reset_control_assert(host->pdata->rstc);
3251 usleep_range(10, 50);
3252 reset_control_deassert(host->pdata->rstc);
3253 }
3254
3255 if (drv_data && drv_data->init) {
3256 ret = drv_data->init(host);
3257 if (ret) {
3258 dev_err(host->dev,
3259 "implementation specific init failed\n");
3260 goto err_clk_ciu;
3261 }
3262 }
3263
3264 timer_setup(&host->cmd11_timer, dw_mci_cmd11_timer, 0);
3265 timer_setup(&host->cto_timer, dw_mci_cto_timer, 0);
3266 timer_setup(&host->dto_timer, dw_mci_dto_timer, 0);
3267
3268 spin_lock_init(&host->lock);
3269 spin_lock_init(&host->irq_lock);
3270 INIT_LIST_HEAD(&host->queue);
3271
3272 /*
3273 * Get the host data width - this assumes that HCON has been set with
3274 * the correct values.
3275 */
3276 i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON));
3277 if (!i) {
3278 host->push_data = dw_mci_push_data16;
3279 host->pull_data = dw_mci_pull_data16;
3280 width = 16;
3281 host->data_shift = 1;
3282 } else if (i == 2) {
3283 host->push_data = dw_mci_push_data64;
3284 host->pull_data = dw_mci_pull_data64;
3285 width = 64;
3286 host->data_shift = 3;
3287 } else {
3288 /* Check for a reserved value, and warn if it is */
3289 WARN((i != 1),
3290 "HCON reports a reserved host data width!\n"
3291 "Defaulting to 32-bit access.\n");
3292 host->push_data = dw_mci_push_data32;
3293 host->pull_data = dw_mci_pull_data32;
3294 width = 32;
3295 host->data_shift = 2;
3296 }
3297
3298 /* Reset all blocks */
3299 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {
3300 ret = -ENODEV;
3301 goto err_clk_ciu;
3302 }
3303
3304 host->dma_ops = host->pdata->dma_ops;
3305 dw_mci_init_dma(host);
3306
3307 /* Clear the interrupts for the host controller */
3308 mci_writel(host, RINTSTS, 0xFFFFFFFF);
3309 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */
3310
3311 /* Put in max timeout */
3312 mci_writel(host, TMOUT, 0xFFFFFFFF);
3313
3314 /*
3315 * FIFO threshold settings RxMark = fifo_size / 2 - 1,
3316 * Tx Mark = fifo_size / 2 DMA Size = 8
3317 */
3318 if (!host->pdata->fifo_depth) {
3319 /*
3320 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may
3321 * have been overwritten by the bootloader, just like we're
3322 * about to do, so if you know the value for your hardware, you
3323 * should put it in the platform data.
3324 */
3325 fifo_size = mci_readl(host, FIFOTH);
3326 fifo_size = 1 + ((fifo_size >> 16) & 0xfff);
3327 } else {
3328 fifo_size = host->pdata->fifo_depth;
3329 }
3330 host->fifo_depth = fifo_size;
3331 host->fifoth_val =
3332 SDMMC_SET_FIFOTH(0x2, fifo_size / 2 - 1, fifo_size / 2);
3333 mci_writel(host, FIFOTH, host->fifoth_val);
3334
3335 /* disable clock to CIU */
3336 mci_writel(host, CLKENA, 0);
3337 mci_writel(host, CLKSRC, 0);
3338
3339 /*
3340 * In 2.40a spec, Data offset is changed.
3341 * Need to check the version-id and set data-offset for DATA register.
3342 */
3343 host->verid = SDMMC_GET_VERID(mci_readl(host, VERID));
3344 dev_info(host->dev, "Version ID is %04x\n", host->verid);
3345
3346 if (host->data_addr_override)
3347 host->fifo_reg = host->regs + host->data_addr_override;
3348 else if (host->verid < DW_MMC_240A)
3349 host->fifo_reg = host->regs + DATA_OFFSET;
3350 else
3351 host->fifo_reg = host->regs + DATA_240A_OFFSET;
3352
3353 tasklet_init(&host->tasklet, dw_mci_tasklet_func, (unsigned long)host);
3354 ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt,
3355 host->irq_flags, "dw-mci", host);
3356 if (ret)
3357 goto err_dmaunmap;
3358
3359 /*
3360 * Enable interrupts for command done, data over, data empty,
3361 * receive ready and error such as transmit, receive timeout, crc error
3362 */
3363 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |
3364 SDMMC_INT_TXDR | SDMMC_INT_RXDR |
3365 DW_MCI_ERROR_FLAGS);
3366 /* Enable mci interrupt */
3367 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);
3368
3369 dev_info(host->dev,
3370 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n",
3371 host->irq, width, fifo_size);
3372
3373 /* We need at least one slot to succeed */
3374 ret = dw_mci_init_slot(host);
3375 if (ret) {
3376 dev_dbg(host->dev, "slot %d init failed\n", i);
3377 goto err_dmaunmap;
3378 }
3379
3380 /* Now that slots are all setup, we can enable card detect */
3381 dw_mci_enable_cd(host);
3382
3383 return 0;
3384
3385 err_dmaunmap:
3386 if (host->use_dma && host->dma_ops->exit)
3387 host->dma_ops->exit(host);
3388
3389 if (!IS_ERR(host->pdata->rstc))
3390 reset_control_assert(host->pdata->rstc);
3391
3392 err_clk_ciu:
3393 clk_disable_unprepare(host->ciu_clk);
3394
3395 err_clk_biu:
3396 clk_disable_unprepare(host->biu_clk);
3397
3398 return ret;
3399 }
3400 EXPORT_SYMBOL(dw_mci_probe);
3401
3402 void dw_mci_remove(struct dw_mci *host)
3403 {
3404 dev_dbg(host->dev, "remove slot\n");
3405 if (host->slot)
3406 dw_mci_cleanup_slot(host->slot);
3407
3408 mci_writel(host, RINTSTS, 0xFFFFFFFF);
3409 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */
3410
3411 /* disable clock to CIU */
3412 mci_writel(host, CLKENA, 0);
3413 mci_writel(host, CLKSRC, 0);
3414
3415 if (host->use_dma && host->dma_ops->exit)
3416 host->dma_ops->exit(host);
3417
3418 if (!IS_ERR(host->pdata->rstc))
3419 reset_control_assert(host->pdata->rstc);
3420
3421 clk_disable_unprepare(host->ciu_clk);
3422 clk_disable_unprepare(host->biu_clk);
3423 }
3424 EXPORT_SYMBOL(dw_mci_remove);
3425
3426
3427
3428 #ifdef CONFIG_PM
3429 int dw_mci_runtime_suspend(struct device *dev)
3430 {
3431 struct dw_mci *host = dev_get_drvdata(dev);
3432
3433 if (host->use_dma && host->dma_ops->exit)
3434 host->dma_ops->exit(host);
3435
3436 clk_disable_unprepare(host->ciu_clk);
3437
3438 if (host->slot &&
3439 (mmc_can_gpio_cd(host->slot->mmc) ||
3440 !mmc_card_is_removable(host->slot->mmc)))
3441 clk_disable_unprepare(host->biu_clk);
3442
3443 return 0;
3444 }
3445 EXPORT_SYMBOL(dw_mci_runtime_suspend);
3446
3447 int dw_mci_runtime_resume(struct device *dev)
3448 {
3449 int ret = 0;
3450 struct dw_mci *host = dev_get_drvdata(dev);
3451
3452 if (host->slot &&
3453 (mmc_can_gpio_cd(host->slot->mmc) ||
3454 !mmc_card_is_removable(host->slot->mmc))) {
3455 ret = clk_prepare_enable(host->biu_clk);
3456 if (ret)
3457 return ret;
3458 }
3459
3460 ret = clk_prepare_enable(host->ciu_clk);
3461 if (ret)
3462 goto err;
3463
3464 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {
3465 clk_disable_unprepare(host->ciu_clk);
3466 ret = -ENODEV;
3467 goto err;
3468 }
3469
3470 if (host->use_dma && host->dma_ops->init)
3471 host->dma_ops->init(host);
3472
3473 /*
3474 * Restore the initial value at FIFOTH register
3475 * And Invalidate the prev_blksz with zero
3476 */
3477 mci_writel(host, FIFOTH, host->fifoth_val);
3478 host->prev_blksz = 0;
3479
3480 /* Put in max timeout */
3481 mci_writel(host, TMOUT, 0xFFFFFFFF);
3482
3483 mci_writel(host, RINTSTS, 0xFFFFFFFF);
3484 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |
3485 SDMMC_INT_TXDR | SDMMC_INT_RXDR |
3486 DW_MCI_ERROR_FLAGS);
3487 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);
3488
3489
3490 if (host->slot->mmc->pm_flags & MMC_PM_KEEP_POWER)
3491 dw_mci_set_ios(host->slot->mmc, &host->slot->mmc->ios);
3492
3493 /* Force setup bus to guarantee available clock output */
3494 dw_mci_setup_bus(host->slot, true);
3495
3496 /* Now that slots are all setup, we can enable card detect */
3497 dw_mci_enable_cd(host);
3498
3499 return 0;
3500
3501 err:
3502 if (host->slot &&
3503 (mmc_can_gpio_cd(host->slot->mmc) ||
3504 !mmc_card_is_removable(host->slot->mmc)))
3505 clk_disable_unprepare(host->biu_clk);
3506
3507 return ret;
3508 }
3509 EXPORT_SYMBOL(dw_mci_runtime_resume);
3510 #endif /* CONFIG_PM */
3511
3512 static int __init dw_mci_init(void)
3513 {
3514 pr_info("Synopsys Designware Multimedia Card Interface Driver\n");
3515 return 0;
3516 }
3517
3518 static void __exit dw_mci_exit(void)
3519 {
3520 }
3521
3522 module_init(dw_mci_init);
3523 module_exit(dw_mci_exit);
3524
3525 MODULE_DESCRIPTION("DW Multimedia Card Interface driver");
3526 MODULE_AUTHOR("NXP Semiconductor VietNam");
3527 MODULE_AUTHOR("Imagination Technologies Ltd");
3528 MODULE_LICENSE("GPL v2");
ubuntu-bionic-kernel mirror