2 * libahci.c - Common AHCI SATA low-level routines
4 * Maintained by: Tejun Heo <tj@kernel.org>
5 * Please ALWAYS copy linux-ide@vger.kernel.org
8 * Copyright 2004-2005 Red Hat, Inc.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; see the file COPYING. If not, write to
23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26 * libata documentation is available via 'make {ps|pdf}docs',
27 * as Documentation/DocBook/libata.*
29 * AHCI hardware documentation:
30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
35 #include <linux/kernel.h>
36 #include <linux/gfp.h>
37 #include <linux/module.h>
38 #include <linux/blkdev.h>
39 #include <linux/delay.h>
40 #include <linux/interrupt.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/device.h>
43 #include <scsi/scsi_host.h>
44 #include <scsi/scsi_cmnd.h>
45 #include <linux/libata.h>
46 #include <linux/pci.h>
50 static int ahci_skip_host_reset
;
52 EXPORT_SYMBOL_GPL(ahci_ignore_sss
);
54 module_param_named(skip_host_reset
, ahci_skip_host_reset
, int, 0444);
55 MODULE_PARM_DESC(skip_host_reset
, "skip global host reset (0=don't skip, 1=skip)");
57 module_param_named(ignore_sss
, ahci_ignore_sss
, int, 0444);
58 MODULE_PARM_DESC(ignore_sss
, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
60 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
62 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
);
63 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
65 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
70 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
);
71 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
);
72 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
);
73 static int ahci_port_start(struct ata_port
*ap
);
74 static void ahci_port_stop(struct ata_port
*ap
);
75 static void ahci_qc_prep(struct ata_queued_cmd
*qc
);
76 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
);
77 static void ahci_freeze(struct ata_port
*ap
);
78 static void ahci_thaw(struct ata_port
*ap
);
79 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
);
80 static void ahci_enable_fbs(struct ata_port
*ap
);
81 static void ahci_disable_fbs(struct ata_port
*ap
);
82 static void ahci_pmp_attach(struct ata_port
*ap
);
83 static void ahci_pmp_detach(struct ata_port
*ap
);
84 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
85 unsigned long deadline
);
86 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
87 unsigned long deadline
);
88 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
89 unsigned long deadline
);
90 static void ahci_postreset(struct ata_link
*link
, unsigned int *class);
91 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
);
92 static void ahci_dev_config(struct ata_device
*dev
);
94 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
);
96 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
);
97 static ssize_t
ahci_activity_store(struct ata_device
*dev
,
98 enum sw_activity val
);
99 static void ahci_init_sw_activity(struct ata_link
*link
);
101 static ssize_t
ahci_show_host_caps(struct device
*dev
,
102 struct device_attribute
*attr
, char *buf
);
103 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
104 struct device_attribute
*attr
, char *buf
);
105 static ssize_t
ahci_show_host_version(struct device
*dev
,
106 struct device_attribute
*attr
, char *buf
);
107 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
108 struct device_attribute
*attr
, char *buf
);
109 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
110 struct device_attribute
*attr
, char *buf
);
111 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
112 struct device_attribute
*attr
,
113 const char *buf
, size_t size
);
114 static ssize_t
ahci_show_em_supported(struct device
*dev
,
115 struct device_attribute
*attr
, char *buf
);
117 static DEVICE_ATTR(ahci_host_caps
, S_IRUGO
, ahci_show_host_caps
, NULL
);
118 static DEVICE_ATTR(ahci_host_cap2
, S_IRUGO
, ahci_show_host_cap2
, NULL
);
119 static DEVICE_ATTR(ahci_host_version
, S_IRUGO
, ahci_show_host_version
, NULL
);
120 static DEVICE_ATTR(ahci_port_cmd
, S_IRUGO
, ahci_show_port_cmd
, NULL
);
121 static DEVICE_ATTR(em_buffer
, S_IWUSR
| S_IRUGO
,
122 ahci_read_em_buffer
, ahci_store_em_buffer
);
123 static DEVICE_ATTR(em_message_supported
, S_IRUGO
, ahci_show_em_supported
, NULL
);
125 struct device_attribute
*ahci_shost_attrs
[] = {
126 &dev_attr_link_power_management_policy
,
127 &dev_attr_em_message_type
,
128 &dev_attr_em_message
,
129 &dev_attr_ahci_host_caps
,
130 &dev_attr_ahci_host_cap2
,
131 &dev_attr_ahci_host_version
,
132 &dev_attr_ahci_port_cmd
,
134 &dev_attr_em_message_supported
,
137 EXPORT_SYMBOL_GPL(ahci_shost_attrs
);
139 struct device_attribute
*ahci_sdev_attrs
[] = {
140 &dev_attr_sw_activity
,
141 &dev_attr_unload_heads
,
144 EXPORT_SYMBOL_GPL(ahci_sdev_attrs
);
146 struct ata_port_operations ahci_ops
= {
147 .inherits
= &sata_pmp_port_ops
,
149 .qc_defer
= ahci_pmp_qc_defer
,
150 .qc_prep
= ahci_qc_prep
,
151 .qc_issue
= ahci_qc_issue
,
152 .qc_fill_rtf
= ahci_qc_fill_rtf
,
154 .freeze
= ahci_freeze
,
156 .softreset
= ahci_softreset
,
157 .hardreset
= ahci_hardreset
,
158 .postreset
= ahci_postreset
,
159 .pmp_softreset
= ahci_softreset
,
160 .error_handler
= ahci_error_handler
,
161 .post_internal_cmd
= ahci_post_internal_cmd
,
162 .dev_config
= ahci_dev_config
,
164 .scr_read
= ahci_scr_read
,
165 .scr_write
= ahci_scr_write
,
166 .pmp_attach
= ahci_pmp_attach
,
167 .pmp_detach
= ahci_pmp_detach
,
169 .set_lpm
= ahci_set_lpm
,
170 .em_show
= ahci_led_show
,
171 .em_store
= ahci_led_store
,
172 .sw_activity_show
= ahci_activity_show
,
173 .sw_activity_store
= ahci_activity_store
,
174 .transmit_led_message
= ahci_transmit_led_message
,
176 .port_suspend
= ahci_port_suspend
,
177 .port_resume
= ahci_port_resume
,
179 .port_start
= ahci_port_start
,
180 .port_stop
= ahci_port_stop
,
182 EXPORT_SYMBOL_GPL(ahci_ops
);
184 struct ata_port_operations ahci_pmp_retry_srst_ops
= {
185 .inherits
= &ahci_ops
,
186 .softreset
= ahci_pmp_retry_softreset
,
188 EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops
);
190 static bool ahci_em_messages __read_mostly
= true;
191 EXPORT_SYMBOL_GPL(ahci_em_messages
);
192 module_param(ahci_em_messages
, bool, 0444);
193 /* add other LED protocol types when they become supported */
194 MODULE_PARM_DESC(ahci_em_messages
,
195 "AHCI Enclosure Management Message control (0 = off, 1 = on)");
197 /* device sleep idle timeout in ms */
198 static int devslp_idle_timeout __read_mostly
= 1000;
199 module_param(devslp_idle_timeout
, int, 0644);
200 MODULE_PARM_DESC(devslp_idle_timeout
, "device sleep idle timeout");
202 static void ahci_enable_ahci(void __iomem
*mmio
)
207 /* turn on AHCI_EN */
208 tmp
= readl(mmio
+ HOST_CTL
);
209 if (tmp
& HOST_AHCI_EN
)
212 /* Some controllers need AHCI_EN to be written multiple times.
213 * Try a few times before giving up.
215 for (i
= 0; i
< 5; i
++) {
217 writel(tmp
, mmio
+ HOST_CTL
);
218 tmp
= readl(mmio
+ HOST_CTL
); /* flush && sanity check */
219 if (tmp
& HOST_AHCI_EN
)
227 static ssize_t
ahci_show_host_caps(struct device
*dev
,
228 struct device_attribute
*attr
, char *buf
)
230 struct Scsi_Host
*shost
= class_to_shost(dev
);
231 struct ata_port
*ap
= ata_shost_to_port(shost
);
232 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
234 return sprintf(buf
, "%x\n", hpriv
->cap
);
237 static ssize_t
ahci_show_host_cap2(struct device
*dev
,
238 struct device_attribute
*attr
, char *buf
)
240 struct Scsi_Host
*shost
= class_to_shost(dev
);
241 struct ata_port
*ap
= ata_shost_to_port(shost
);
242 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
244 return sprintf(buf
, "%x\n", hpriv
->cap2
);
247 static ssize_t
ahci_show_host_version(struct device
*dev
,
248 struct device_attribute
*attr
, char *buf
)
250 struct Scsi_Host
*shost
= class_to_shost(dev
);
251 struct ata_port
*ap
= ata_shost_to_port(shost
);
252 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
253 void __iomem
*mmio
= hpriv
->mmio
;
255 return sprintf(buf
, "%x\n", readl(mmio
+ HOST_VERSION
));
258 static ssize_t
ahci_show_port_cmd(struct device
*dev
,
259 struct device_attribute
*attr
, char *buf
)
261 struct Scsi_Host
*shost
= class_to_shost(dev
);
262 struct ata_port
*ap
= ata_shost_to_port(shost
);
263 void __iomem
*port_mmio
= ahci_port_base(ap
);
265 return sprintf(buf
, "%x\n", readl(port_mmio
+ PORT_CMD
));
268 static ssize_t
ahci_read_em_buffer(struct device
*dev
,
269 struct device_attribute
*attr
, char *buf
)
271 struct Scsi_Host
*shost
= class_to_shost(dev
);
272 struct ata_port
*ap
= ata_shost_to_port(shost
);
273 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
274 void __iomem
*mmio
= hpriv
->mmio
;
275 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
281 spin_lock_irqsave(ap
->lock
, flags
);
283 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
284 if (!(ap
->flags
& ATA_FLAG_EM
) || em_ctl
& EM_CTL_XMT
||
285 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
)) {
286 spin_unlock_irqrestore(ap
->lock
, flags
);
290 if (!(em_ctl
& EM_CTL_MR
)) {
291 spin_unlock_irqrestore(ap
->lock
, flags
);
295 if (!(em_ctl
& EM_CTL_SMB
))
296 em_mmio
+= hpriv
->em_buf_sz
;
298 count
= hpriv
->em_buf_sz
;
300 /* the count should not be larger than PAGE_SIZE */
301 if (count
> PAGE_SIZE
) {
302 if (printk_ratelimit())
304 "EM read buffer size too large: "
305 "buffer size %u, page size %lu\n",
306 hpriv
->em_buf_sz
, PAGE_SIZE
);
310 for (i
= 0; i
< count
; i
+= 4) {
311 msg
= readl(em_mmio
+ i
);
313 buf
[i
+ 1] = (msg
>> 8) & 0xff;
314 buf
[i
+ 2] = (msg
>> 16) & 0xff;
315 buf
[i
+ 3] = (msg
>> 24) & 0xff;
318 spin_unlock_irqrestore(ap
->lock
, flags
);
323 static ssize_t
ahci_store_em_buffer(struct device
*dev
,
324 struct device_attribute
*attr
,
325 const char *buf
, size_t size
)
327 struct Scsi_Host
*shost
= class_to_shost(dev
);
328 struct ata_port
*ap
= ata_shost_to_port(shost
);
329 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
330 void __iomem
*mmio
= hpriv
->mmio
;
331 void __iomem
*em_mmio
= mmio
+ hpriv
->em_loc
;
332 const unsigned char *msg_buf
= buf
;
337 /* check size validity */
338 if (!(ap
->flags
& ATA_FLAG_EM
) ||
339 !(hpriv
->em_msg_type
& EM_MSG_TYPE_SGPIO
) ||
340 size
% 4 || size
> hpriv
->em_buf_sz
)
343 spin_lock_irqsave(ap
->lock
, flags
);
345 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
346 if (em_ctl
& EM_CTL_TM
) {
347 spin_unlock_irqrestore(ap
->lock
, flags
);
351 for (i
= 0; i
< size
; i
+= 4) {
352 msg
= msg_buf
[i
] | msg_buf
[i
+ 1] << 8 |
353 msg_buf
[i
+ 2] << 16 | msg_buf
[i
+ 3] << 24;
354 writel(msg
, em_mmio
+ i
);
357 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
359 spin_unlock_irqrestore(ap
->lock
, flags
);
364 static ssize_t
ahci_show_em_supported(struct device
*dev
,
365 struct device_attribute
*attr
, char *buf
)
367 struct Scsi_Host
*shost
= class_to_shost(dev
);
368 struct ata_port
*ap
= ata_shost_to_port(shost
);
369 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
370 void __iomem
*mmio
= hpriv
->mmio
;
373 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
375 return sprintf(buf
, "%s%s%s%s\n",
376 em_ctl
& EM_CTL_LED
? "led " : "",
377 em_ctl
& EM_CTL_SAFTE
? "saf-te " : "",
378 em_ctl
& EM_CTL_SES
? "ses-2 " : "",
379 em_ctl
& EM_CTL_SGPIO
? "sgpio " : "");
383 * ahci_save_initial_config - Save and fixup initial config values
384 * @dev: target AHCI device
385 * @hpriv: host private area to store config values
387 * Some registers containing configuration info might be setup by
388 * BIOS and might be cleared on reset. This function saves the
389 * initial values of those registers into @hpriv such that they
390 * can be restored after controller reset.
392 * If inconsistent, config values are fixed up by this function.
394 * If it is not set already this function sets hpriv->start_engine to
400 void ahci_save_initial_config(struct device
*dev
, struct ahci_host_priv
*hpriv
)
402 void __iomem
*mmio
= hpriv
->mmio
;
403 u32 cap
, cap2
, vers
, port_map
;
406 /* make sure AHCI mode is enabled before accessing CAP */
407 ahci_enable_ahci(mmio
);
409 /* Values prefixed with saved_ are written back to host after
410 * reset. Values without are used for driver operation.
412 hpriv
->saved_cap
= cap
= readl(mmio
+ HOST_CAP
);
413 hpriv
->saved_port_map
= port_map
= readl(mmio
+ HOST_PORTS_IMPL
);
415 /* CAP2 register is only defined for AHCI 1.2 and later */
416 vers
= readl(mmio
+ HOST_VERSION
);
417 if ((vers
>> 16) > 1 ||
418 ((vers
>> 16) == 1 && (vers
& 0xFFFF) >= 0x200))
419 hpriv
->saved_cap2
= cap2
= readl(mmio
+ HOST_CAP2
);
421 hpriv
->saved_cap2
= cap2
= 0;
423 /* some chips have errata preventing 64bit use */
424 if ((cap
& HOST_CAP_64
) && (hpriv
->flags
& AHCI_HFLAG_32BIT_ONLY
)) {
425 dev_info(dev
, "controller can't do 64bit DMA, forcing 32bit\n");
429 if ((cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_NO_NCQ
)) {
430 dev_info(dev
, "controller can't do NCQ, turning off CAP_NCQ\n");
431 cap
&= ~HOST_CAP_NCQ
;
434 if (!(cap
& HOST_CAP_NCQ
) && (hpriv
->flags
& AHCI_HFLAG_YES_NCQ
)) {
435 dev_info(dev
, "controller can do NCQ, turning on CAP_NCQ\n");
439 if ((cap
& HOST_CAP_PMP
) && (hpriv
->flags
& AHCI_HFLAG_NO_PMP
)) {
440 dev_info(dev
, "controller can't do PMP, turning off CAP_PMP\n");
441 cap
&= ~HOST_CAP_PMP
;
444 if ((cap
& HOST_CAP_SNTF
) && (hpriv
->flags
& AHCI_HFLAG_NO_SNTF
)) {
446 "controller can't do SNTF, turning off CAP_SNTF\n");
447 cap
&= ~HOST_CAP_SNTF
;
450 if ((cap2
& HOST_CAP2_SDS
) && (hpriv
->flags
& AHCI_HFLAG_NO_DEVSLP
)) {
452 "controller can't do DEVSLP, turning off\n");
453 cap2
&= ~HOST_CAP2_SDS
;
454 cap2
&= ~HOST_CAP2_SADM
;
457 if (!(cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_YES_FBS
)) {
458 dev_info(dev
, "controller can do FBS, turning on CAP_FBS\n");
462 if ((cap
& HOST_CAP_FBS
) && (hpriv
->flags
& AHCI_HFLAG_NO_FBS
)) {
463 dev_info(dev
, "controller can't do FBS, turning off CAP_FBS\n");
464 cap
&= ~HOST_CAP_FBS
;
467 if (hpriv
->force_port_map
&& port_map
!= hpriv
->force_port_map
) {
468 dev_info(dev
, "forcing port_map 0x%x -> 0x%x\n",
469 port_map
, hpriv
->force_port_map
);
470 port_map
= hpriv
->force_port_map
;
473 if (hpriv
->mask_port_map
) {
474 dev_warn(dev
, "masking port_map 0x%x -> 0x%x\n",
476 port_map
& hpriv
->mask_port_map
);
477 port_map
&= hpriv
->mask_port_map
;
480 /* cross check port_map and cap.n_ports */
484 for (i
= 0; i
< AHCI_MAX_PORTS
; i
++)
485 if (port_map
& (1 << i
))
488 /* If PI has more ports than n_ports, whine, clear
489 * port_map and let it be generated from n_ports.
491 if (map_ports
> ahci_nr_ports(cap
)) {
493 "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
494 port_map
, ahci_nr_ports(cap
));
499 /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
500 if (!port_map
&& vers
< 0x10300) {
501 port_map
= (1 << ahci_nr_ports(cap
)) - 1;
502 dev_warn(dev
, "forcing PORTS_IMPL to 0x%x\n", port_map
);
504 /* write the fixed up value to the PI register */
505 hpriv
->saved_port_map
= port_map
;
508 /* record values to use during operation */
511 hpriv
->port_map
= port_map
;
513 if (!hpriv
->start_engine
)
514 hpriv
->start_engine
= ahci_start_engine
;
516 EXPORT_SYMBOL_GPL(ahci_save_initial_config
);
519 * ahci_restore_initial_config - Restore initial config
520 * @host: target ATA host
522 * Restore initial config stored by ahci_save_initial_config().
527 static void ahci_restore_initial_config(struct ata_host
*host
)
529 struct ahci_host_priv
*hpriv
= host
->private_data
;
530 void __iomem
*mmio
= hpriv
->mmio
;
532 writel(hpriv
->saved_cap
, mmio
+ HOST_CAP
);
533 if (hpriv
->saved_cap2
)
534 writel(hpriv
->saved_cap2
, mmio
+ HOST_CAP2
);
535 writel(hpriv
->saved_port_map
, mmio
+ HOST_PORTS_IMPL
);
536 (void) readl(mmio
+ HOST_PORTS_IMPL
); /* flush */
539 static unsigned ahci_scr_offset(struct ata_port
*ap
, unsigned int sc_reg
)
541 static const int offset
[] = {
542 [SCR_STATUS
] = PORT_SCR_STAT
,
543 [SCR_CONTROL
] = PORT_SCR_CTL
,
544 [SCR_ERROR
] = PORT_SCR_ERR
,
545 [SCR_ACTIVE
] = PORT_SCR_ACT
,
546 [SCR_NOTIFICATION
] = PORT_SCR_NTF
,
548 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
550 if (sc_reg
< ARRAY_SIZE(offset
) &&
551 (sc_reg
!= SCR_NOTIFICATION
|| (hpriv
->cap
& HOST_CAP_SNTF
)))
552 return offset
[sc_reg
];
556 static int ahci_scr_read(struct ata_link
*link
, unsigned int sc_reg
, u32
*val
)
558 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
559 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
562 *val
= readl(port_mmio
+ offset
);
568 static int ahci_scr_write(struct ata_link
*link
, unsigned int sc_reg
, u32 val
)
570 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
571 int offset
= ahci_scr_offset(link
->ap
, sc_reg
);
574 writel(val
, port_mmio
+ offset
);
580 void ahci_start_engine(struct ata_port
*ap
)
582 void __iomem
*port_mmio
= ahci_port_base(ap
);
586 tmp
= readl(port_mmio
+ PORT_CMD
);
587 tmp
|= PORT_CMD_START
;
588 writel(tmp
, port_mmio
+ PORT_CMD
);
589 readl(port_mmio
+ PORT_CMD
); /* flush */
591 EXPORT_SYMBOL_GPL(ahci_start_engine
);
593 int ahci_stop_engine(struct ata_port
*ap
)
595 void __iomem
*port_mmio
= ahci_port_base(ap
);
596 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
600 * On some controllers, stopping a port's DMA engine while the port
601 * is in ALPM state (partial or slumber) results in failures on
602 * subsequent DMA engine starts. For those controllers, put the
603 * port back in active state before stopping its DMA engine.
605 if ((hpriv
->flags
& AHCI_HFLAG_WAKE_BEFORE_STOP
) &&
606 (ap
->link
.lpm_policy
> ATA_LPM_MAX_POWER
) &&
607 ahci_set_lpm(&ap
->link
, ATA_LPM_MAX_POWER
, ATA_LPM_WAKE_ONLY
)) {
608 dev_err(ap
->host
->dev
, "Failed to wake up port before engine stop\n");
612 tmp
= readl(port_mmio
+ PORT_CMD
);
614 /* check if the HBA is idle */
615 if ((tmp
& (PORT_CMD_START
| PORT_CMD_LIST_ON
)) == 0)
618 /* setting HBA to idle */
619 tmp
&= ~PORT_CMD_START
;
620 writel(tmp
, port_mmio
+ PORT_CMD
);
622 /* wait for engine to stop. This could be as long as 500 msec */
623 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
624 PORT_CMD_LIST_ON
, PORT_CMD_LIST_ON
, 1, 500);
625 if (tmp
& PORT_CMD_LIST_ON
)
630 EXPORT_SYMBOL_GPL(ahci_stop_engine
);
632 void ahci_start_fis_rx(struct ata_port
*ap
)
634 void __iomem
*port_mmio
= ahci_port_base(ap
);
635 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
636 struct ahci_port_priv
*pp
= ap
->private_data
;
639 /* set FIS registers */
640 if (hpriv
->cap
& HOST_CAP_64
)
641 writel((pp
->cmd_slot_dma
>> 16) >> 16,
642 port_mmio
+ PORT_LST_ADDR_HI
);
643 writel(pp
->cmd_slot_dma
& 0xffffffff, port_mmio
+ PORT_LST_ADDR
);
645 if (hpriv
->cap
& HOST_CAP_64
)
646 writel((pp
->rx_fis_dma
>> 16) >> 16,
647 port_mmio
+ PORT_FIS_ADDR_HI
);
648 writel(pp
->rx_fis_dma
& 0xffffffff, port_mmio
+ PORT_FIS_ADDR
);
650 /* enable FIS reception */
651 tmp
= readl(port_mmio
+ PORT_CMD
);
652 tmp
|= PORT_CMD_FIS_RX
;
653 writel(tmp
, port_mmio
+ PORT_CMD
);
656 readl(port_mmio
+ PORT_CMD
);
658 EXPORT_SYMBOL_GPL(ahci_start_fis_rx
);
660 static int ahci_stop_fis_rx(struct ata_port
*ap
)
662 void __iomem
*port_mmio
= ahci_port_base(ap
);
665 /* disable FIS reception */
666 tmp
= readl(port_mmio
+ PORT_CMD
);
667 tmp
&= ~PORT_CMD_FIS_RX
;
668 writel(tmp
, port_mmio
+ PORT_CMD
);
670 /* wait for completion, spec says 500ms, give it 1000 */
671 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
, PORT_CMD_FIS_ON
,
672 PORT_CMD_FIS_ON
, 10, 1000);
673 if (tmp
& PORT_CMD_FIS_ON
)
679 static void ahci_power_up(struct ata_port
*ap
)
681 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
682 void __iomem
*port_mmio
= ahci_port_base(ap
);
685 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
688 if (hpriv
->cap
& HOST_CAP_SSS
) {
689 cmd
|= PORT_CMD_SPIN_UP
;
690 writel(cmd
, port_mmio
+ PORT_CMD
);
694 writel(cmd
| PORT_CMD_ICC_ACTIVE
, port_mmio
+ PORT_CMD
);
697 static int ahci_set_lpm(struct ata_link
*link
, enum ata_lpm_policy policy
,
700 struct ata_port
*ap
= link
->ap
;
701 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
702 struct ahci_port_priv
*pp
= ap
->private_data
;
703 void __iomem
*port_mmio
= ahci_port_base(ap
);
705 if (policy
!= ATA_LPM_MAX_POWER
) {
706 /* wakeup flag only applies to the max power policy */
707 hints
&= ~ATA_LPM_WAKE_ONLY
;
710 * Disable interrupts on Phy Ready. This keeps us from
711 * getting woken up due to spurious phy ready
714 pp
->intr_mask
&= ~PORT_IRQ_PHYRDY
;
715 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
717 sata_link_scr_lpm(link
, policy
, false);
720 if (hpriv
->cap
& HOST_CAP_ALPM
) {
721 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
723 if (policy
== ATA_LPM_MAX_POWER
|| !(hints
& ATA_LPM_HIPM
)) {
724 if (!(hints
& ATA_LPM_WAKE_ONLY
))
725 cmd
&= ~(PORT_CMD_ASP
| PORT_CMD_ALPE
);
726 cmd
|= PORT_CMD_ICC_ACTIVE
;
728 writel(cmd
, port_mmio
+ PORT_CMD
);
729 readl(port_mmio
+ PORT_CMD
);
731 /* wait 10ms to be sure we've come out of LPM state */
734 if (hints
& ATA_LPM_WAKE_ONLY
)
737 cmd
|= PORT_CMD_ALPE
;
738 if (policy
== ATA_LPM_MIN_POWER
)
741 /* write out new cmd value */
742 writel(cmd
, port_mmio
+ PORT_CMD
);
746 /* set aggressive device sleep */
747 if ((hpriv
->cap2
& HOST_CAP2_SDS
) &&
748 (hpriv
->cap2
& HOST_CAP2_SADM
) &&
749 (link
->device
->flags
& ATA_DFLAG_DEVSLP
)) {
750 if (policy
== ATA_LPM_MIN_POWER
)
751 ahci_set_aggressive_devslp(ap
, true);
753 ahci_set_aggressive_devslp(ap
, false);
756 if (policy
== ATA_LPM_MAX_POWER
) {
757 sata_link_scr_lpm(link
, policy
, false);
759 /* turn PHYRDY IRQ back on */
760 pp
->intr_mask
|= PORT_IRQ_PHYRDY
;
761 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
768 static void ahci_power_down(struct ata_port
*ap
)
770 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
771 void __iomem
*port_mmio
= ahci_port_base(ap
);
774 if (!(hpriv
->cap
& HOST_CAP_SSS
))
777 /* put device into listen mode, first set PxSCTL.DET to 0 */
778 scontrol
= readl(port_mmio
+ PORT_SCR_CTL
);
780 writel(scontrol
, port_mmio
+ PORT_SCR_CTL
);
782 /* then set PxCMD.SUD to 0 */
783 cmd
= readl(port_mmio
+ PORT_CMD
) & ~PORT_CMD_ICC_MASK
;
784 cmd
&= ~PORT_CMD_SPIN_UP
;
785 writel(cmd
, port_mmio
+ PORT_CMD
);
789 static void ahci_start_port(struct ata_port
*ap
)
791 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
792 struct ahci_port_priv
*pp
= ap
->private_data
;
793 struct ata_link
*link
;
794 struct ahci_em_priv
*emp
;
798 /* enable FIS reception */
799 ahci_start_fis_rx(ap
);
802 if (!(hpriv
->flags
& AHCI_HFLAG_DELAY_ENGINE
))
803 hpriv
->start_engine(ap
);
806 if (ap
->flags
& ATA_FLAG_EM
) {
807 ata_for_each_link(link
, ap
, EDGE
) {
808 emp
= &pp
->em_priv
[link
->pmp
];
810 /* EM Transmit bit maybe busy during init */
811 for (i
= 0; i
< EM_MAX_RETRY
; i
++) {
812 rc
= ap
->ops
->transmit_led_message(ap
,
816 * If busy, give a breather but do not
817 * release EH ownership by using msleep()
818 * instead of ata_msleep(). EM Transmit
819 * bit is busy for the whole host and
820 * releasing ownership will cause other
821 * ports to fail the same way.
831 if (ap
->flags
& ATA_FLAG_SW_ACTIVITY
)
832 ata_for_each_link(link
, ap
, EDGE
)
833 ahci_init_sw_activity(link
);
837 static int ahci_deinit_port(struct ata_port
*ap
, const char **emsg
)
842 rc
= ahci_stop_engine(ap
);
844 *emsg
= "failed to stop engine";
848 /* disable FIS reception */
849 rc
= ahci_stop_fis_rx(ap
);
851 *emsg
= "failed stop FIS RX";
858 int ahci_reset_controller(struct ata_host
*host
)
860 struct ahci_host_priv
*hpriv
= host
->private_data
;
861 void __iomem
*mmio
= hpriv
->mmio
;
864 /* we must be in AHCI mode, before using anything
865 * AHCI-specific, such as HOST_RESET.
867 ahci_enable_ahci(mmio
);
869 /* global controller reset */
870 if (!ahci_skip_host_reset
) {
871 tmp
= readl(mmio
+ HOST_CTL
);
872 if ((tmp
& HOST_RESET
) == 0) {
873 writel(tmp
| HOST_RESET
, mmio
+ HOST_CTL
);
874 readl(mmio
+ HOST_CTL
); /* flush */
878 * to perform host reset, OS should set HOST_RESET
879 * and poll until this bit is read to be "0".
880 * reset must complete within 1 second, or
881 * the hardware should be considered fried.
883 tmp
= ata_wait_register(NULL
, mmio
+ HOST_CTL
, HOST_RESET
,
884 HOST_RESET
, 10, 1000);
886 if (tmp
& HOST_RESET
) {
887 dev_err(host
->dev
, "controller reset failed (0x%x)\n",
892 /* turn on AHCI mode */
893 ahci_enable_ahci(mmio
);
895 /* Some registers might be cleared on reset. Restore
898 ahci_restore_initial_config(host
);
900 dev_info(host
->dev
, "skipping global host reset\n");
904 EXPORT_SYMBOL_GPL(ahci_reset_controller
);
906 static void ahci_sw_activity(struct ata_link
*link
)
908 struct ata_port
*ap
= link
->ap
;
909 struct ahci_port_priv
*pp
= ap
->private_data
;
910 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
912 if (!(link
->flags
& ATA_LFLAG_SW_ACTIVITY
))
916 if (!timer_pending(&emp
->timer
))
917 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(10));
920 static void ahci_sw_activity_blink(unsigned long arg
)
922 struct ata_link
*link
= (struct ata_link
*)arg
;
923 struct ata_port
*ap
= link
->ap
;
924 struct ahci_port_priv
*pp
= ap
->private_data
;
925 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
926 unsigned long led_message
= emp
->led_state
;
927 u32 activity_led_state
;
930 led_message
&= EM_MSG_LED_VALUE
;
931 led_message
|= ap
->port_no
| (link
->pmp
<< 8);
933 /* check to see if we've had activity. If so,
934 * toggle state of LED and reset timer. If not,
935 * turn LED to desired idle state.
937 spin_lock_irqsave(ap
->lock
, flags
);
938 if (emp
->saved_activity
!= emp
->activity
) {
939 emp
->saved_activity
= emp
->activity
;
940 /* get the current LED state */
941 activity_led_state
= led_message
& EM_MSG_LED_VALUE_ON
;
943 if (activity_led_state
)
944 activity_led_state
= 0;
946 activity_led_state
= 1;
948 /* clear old state */
949 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
952 led_message
|= (activity_led_state
<< 16);
953 mod_timer(&emp
->timer
, jiffies
+ msecs_to_jiffies(100));
956 led_message
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
957 if (emp
->blink_policy
== BLINK_OFF
)
958 led_message
|= (1 << 16);
960 spin_unlock_irqrestore(ap
->lock
, flags
);
961 ap
->ops
->transmit_led_message(ap
, led_message
, 4);
964 static void ahci_init_sw_activity(struct ata_link
*link
)
966 struct ata_port
*ap
= link
->ap
;
967 struct ahci_port_priv
*pp
= ap
->private_data
;
968 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
970 /* init activity stats, setup timer */
971 emp
->saved_activity
= emp
->activity
= 0;
972 setup_timer(&emp
->timer
, ahci_sw_activity_blink
, (unsigned long)link
);
974 /* check our blink policy and set flag for link if it's enabled */
975 if (emp
->blink_policy
)
976 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
979 int ahci_reset_em(struct ata_host
*host
)
981 struct ahci_host_priv
*hpriv
= host
->private_data
;
982 void __iomem
*mmio
= hpriv
->mmio
;
985 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
986 if ((em_ctl
& EM_CTL_TM
) || (em_ctl
& EM_CTL_RST
))
989 writel(em_ctl
| EM_CTL_RST
, mmio
+ HOST_EM_CTL
);
992 EXPORT_SYMBOL_GPL(ahci_reset_em
);
994 static ssize_t
ahci_transmit_led_message(struct ata_port
*ap
, u32 state
,
997 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
998 struct ahci_port_priv
*pp
= ap
->private_data
;
999 void __iomem
*mmio
= hpriv
->mmio
;
1001 u32 message
[] = {0, 0};
1002 unsigned long flags
;
1004 struct ahci_em_priv
*emp
;
1006 /* get the slot number from the message */
1007 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
1008 if (pmp
< EM_MAX_SLOTS
)
1009 emp
= &pp
->em_priv
[pmp
];
1013 spin_lock_irqsave(ap
->lock
, flags
);
1016 * if we are still busy transmitting a previous message,
1019 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
1020 if (em_ctl
& EM_CTL_TM
) {
1021 spin_unlock_irqrestore(ap
->lock
, flags
);
1025 if (hpriv
->em_msg_type
& EM_MSG_TYPE_LED
) {
1027 * create message header - this is all zero except for
1028 * the message size, which is 4 bytes.
1030 message
[0] |= (4 << 8);
1032 /* ignore 0:4 of byte zero, fill in port info yourself */
1033 message
[1] = ((state
& ~EM_MSG_LED_HBA_PORT
) | ap
->port_no
);
1035 /* write message to EM_LOC */
1036 writel(message
[0], mmio
+ hpriv
->em_loc
);
1037 writel(message
[1], mmio
+ hpriv
->em_loc
+4);
1040 * tell hardware to transmit the message
1042 writel(em_ctl
| EM_CTL_TM
, mmio
+ HOST_EM_CTL
);
1045 /* save off new led state for port/slot */
1046 emp
->led_state
= state
;
1048 spin_unlock_irqrestore(ap
->lock
, flags
);
1052 static ssize_t
ahci_led_show(struct ata_port
*ap
, char *buf
)
1054 struct ahci_port_priv
*pp
= ap
->private_data
;
1055 struct ata_link
*link
;
1056 struct ahci_em_priv
*emp
;
1059 ata_for_each_link(link
, ap
, EDGE
) {
1060 emp
= &pp
->em_priv
[link
->pmp
];
1061 rc
+= sprintf(buf
, "%lx\n", emp
->led_state
);
1066 static ssize_t
ahci_led_store(struct ata_port
*ap
, const char *buf
,
1071 struct ahci_port_priv
*pp
= ap
->private_data
;
1072 struct ahci_em_priv
*emp
;
1074 if (kstrtouint(buf
, 0, &state
) < 0)
1077 /* get the slot number from the message */
1078 pmp
= (state
& EM_MSG_LED_PMP_SLOT
) >> 8;
1079 if (pmp
< EM_MAX_SLOTS
)
1080 emp
= &pp
->em_priv
[pmp
];
1084 /* mask off the activity bits if we are in sw_activity
1085 * mode, user should turn off sw_activity before setting
1086 * activity led through em_message
1088 if (emp
->blink_policy
)
1089 state
&= ~EM_MSG_LED_VALUE_ACTIVITY
;
1091 return ap
->ops
->transmit_led_message(ap
, state
, size
);
1094 static ssize_t
ahci_activity_store(struct ata_device
*dev
, enum sw_activity val
)
1096 struct ata_link
*link
= dev
->link
;
1097 struct ata_port
*ap
= link
->ap
;
1098 struct ahci_port_priv
*pp
= ap
->private_data
;
1099 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1100 u32 port_led_state
= emp
->led_state
;
1102 /* save the desired Activity LED behavior */
1105 link
->flags
&= ~(ATA_LFLAG_SW_ACTIVITY
);
1107 /* set the LED to OFF */
1108 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1109 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1110 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1112 link
->flags
|= ATA_LFLAG_SW_ACTIVITY
;
1113 if (val
== BLINK_OFF
) {
1114 /* set LED to ON for idle */
1115 port_led_state
&= EM_MSG_LED_VALUE_OFF
;
1116 port_led_state
|= (ap
->port_no
| (link
->pmp
<< 8));
1117 port_led_state
|= EM_MSG_LED_VALUE_ON
; /* check this */
1118 ap
->ops
->transmit_led_message(ap
, port_led_state
, 4);
1121 emp
->blink_policy
= val
;
1125 static ssize_t
ahci_activity_show(struct ata_device
*dev
, char *buf
)
1127 struct ata_link
*link
= dev
->link
;
1128 struct ata_port
*ap
= link
->ap
;
1129 struct ahci_port_priv
*pp
= ap
->private_data
;
1130 struct ahci_em_priv
*emp
= &pp
->em_priv
[link
->pmp
];
1132 /* display the saved value of activity behavior for this
1135 return sprintf(buf
, "%d\n", emp
->blink_policy
);
1138 static void ahci_port_init(struct device
*dev
, struct ata_port
*ap
,
1139 int port_no
, void __iomem
*mmio
,
1140 void __iomem
*port_mmio
)
1142 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1143 const char *emsg
= NULL
;
1147 /* make sure port is not active */
1148 rc
= ahci_deinit_port(ap
, &emsg
);
1150 dev_warn(dev
, "%s (%d)\n", emsg
, rc
);
1153 tmp
= readl(port_mmio
+ PORT_SCR_ERR
);
1154 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp
);
1155 writel(tmp
, port_mmio
+ PORT_SCR_ERR
);
1157 /* clear port IRQ */
1158 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
1159 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp
);
1161 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
1163 writel(1 << port_no
, mmio
+ HOST_IRQ_STAT
);
1165 /* mark esata ports */
1166 tmp
= readl(port_mmio
+ PORT_CMD
);
1167 if ((tmp
& PORT_CMD_HPCP
) ||
1168 ((tmp
& PORT_CMD_ESP
) && (hpriv
->cap
& HOST_CAP_SXS
)))
1169 ap
->pflags
|= ATA_PFLAG_EXTERNAL
;
1172 void ahci_init_controller(struct ata_host
*host
)
1174 struct ahci_host_priv
*hpriv
= host
->private_data
;
1175 void __iomem
*mmio
= hpriv
->mmio
;
1177 void __iomem
*port_mmio
;
1180 for (i
= 0; i
< host
->n_ports
; i
++) {
1181 struct ata_port
*ap
= host
->ports
[i
];
1183 port_mmio
= ahci_port_base(ap
);
1184 if (ata_port_is_dummy(ap
))
1187 ahci_port_init(host
->dev
, ap
, i
, mmio
, port_mmio
);
1190 tmp
= readl(mmio
+ HOST_CTL
);
1191 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1192 writel(tmp
| HOST_IRQ_EN
, mmio
+ HOST_CTL
);
1193 tmp
= readl(mmio
+ HOST_CTL
);
1194 VPRINTK("HOST_CTL 0x%x\n", tmp
);
1196 EXPORT_SYMBOL_GPL(ahci_init_controller
);
1198 static void ahci_dev_config(struct ata_device
*dev
)
1200 struct ahci_host_priv
*hpriv
= dev
->link
->ap
->host
->private_data
;
1202 if (hpriv
->flags
& AHCI_HFLAG_SECT255
) {
1203 dev
->max_sectors
= 255;
1205 "SB600 AHCI: limiting to 255 sectors per cmd\n");
1209 unsigned int ahci_dev_classify(struct ata_port
*ap
)
1211 void __iomem
*port_mmio
= ahci_port_base(ap
);
1212 struct ata_taskfile tf
;
1215 tmp
= readl(port_mmio
+ PORT_SIG
);
1216 tf
.lbah
= (tmp
>> 24) & 0xff;
1217 tf
.lbam
= (tmp
>> 16) & 0xff;
1218 tf
.lbal
= (tmp
>> 8) & 0xff;
1219 tf
.nsect
= (tmp
) & 0xff;
1221 return ata_dev_classify(&tf
);
1223 EXPORT_SYMBOL_GPL(ahci_dev_classify
);
1225 void ahci_fill_cmd_slot(struct ahci_port_priv
*pp
, unsigned int tag
,
1228 dma_addr_t cmd_tbl_dma
;
1230 cmd_tbl_dma
= pp
->cmd_tbl_dma
+ tag
* AHCI_CMD_TBL_SZ
;
1232 pp
->cmd_slot
[tag
].opts
= cpu_to_le32(opts
);
1233 pp
->cmd_slot
[tag
].status
= 0;
1234 pp
->cmd_slot
[tag
].tbl_addr
= cpu_to_le32(cmd_tbl_dma
& 0xffffffff);
1235 pp
->cmd_slot
[tag
].tbl_addr_hi
= cpu_to_le32((cmd_tbl_dma
>> 16) >> 16);
1237 EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot
);
1239 int ahci_kick_engine(struct ata_port
*ap
)
1241 void __iomem
*port_mmio
= ahci_port_base(ap
);
1242 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1243 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1248 rc
= ahci_stop_engine(ap
);
1253 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1255 busy
= status
& (ATA_BUSY
| ATA_DRQ
);
1256 if (!busy
&& !sata_pmp_attached(ap
)) {
1261 if (!(hpriv
->cap
& HOST_CAP_CLO
)) {
1267 tmp
= readl(port_mmio
+ PORT_CMD
);
1268 tmp
|= PORT_CMD_CLO
;
1269 writel(tmp
, port_mmio
+ PORT_CMD
);
1272 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD
,
1273 PORT_CMD_CLO
, PORT_CMD_CLO
, 1, 500);
1274 if (tmp
& PORT_CMD_CLO
)
1277 /* restart engine */
1279 hpriv
->start_engine(ap
);
1282 EXPORT_SYMBOL_GPL(ahci_kick_engine
);
1284 static int ahci_exec_polled_cmd(struct ata_port
*ap
, int pmp
,
1285 struct ata_taskfile
*tf
, int is_cmd
, u16 flags
,
1286 unsigned long timeout_msec
)
1288 const u32 cmd_fis_len
= 5; /* five dwords */
1289 struct ahci_port_priv
*pp
= ap
->private_data
;
1290 void __iomem
*port_mmio
= ahci_port_base(ap
);
1291 u8
*fis
= pp
->cmd_tbl
;
1294 /* prep the command */
1295 ata_tf_to_fis(tf
, pmp
, is_cmd
, fis
);
1296 ahci_fill_cmd_slot(pp
, 0, cmd_fis_len
| flags
| (pmp
<< 12));
1298 /* set port value for softreset of Port Multiplier */
1299 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= pmp
) {
1300 tmp
= readl(port_mmio
+ PORT_FBS
);
1301 tmp
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
1302 tmp
|= pmp
<< PORT_FBS_DEV_OFFSET
;
1303 writel(tmp
, port_mmio
+ PORT_FBS
);
1304 pp
->fbs_last_dev
= pmp
;
1308 writel(1, port_mmio
+ PORT_CMD_ISSUE
);
1311 tmp
= ata_wait_register(ap
, port_mmio
+ PORT_CMD_ISSUE
,
1312 0x1, 0x1, 1, timeout_msec
);
1314 ahci_kick_engine(ap
);
1318 readl(port_mmio
+ PORT_CMD_ISSUE
); /* flush */
1323 int ahci_do_softreset(struct ata_link
*link
, unsigned int *class,
1324 int pmp
, unsigned long deadline
,
1325 int (*check_ready
)(struct ata_link
*link
))
1327 struct ata_port
*ap
= link
->ap
;
1328 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1329 struct ahci_port_priv
*pp
= ap
->private_data
;
1330 const char *reason
= NULL
;
1331 unsigned long now
, msecs
;
1332 struct ata_taskfile tf
;
1333 bool fbs_disabled
= false;
1338 /* prepare for SRST (AHCI-1.1 10.4.1) */
1339 rc
= ahci_kick_engine(ap
);
1340 if (rc
&& rc
!= -EOPNOTSUPP
)
1341 ata_link_warn(link
, "failed to reset engine (errno=%d)\n", rc
);
1344 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1345 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1346 * that is attached to port multiplier.
1348 if (!ata_is_host_link(link
) && pp
->fbs_enabled
) {
1349 ahci_disable_fbs(ap
);
1350 fbs_disabled
= true;
1353 ata_tf_init(link
->device
, &tf
);
1355 /* issue the first D2H Register FIS */
1358 if (time_after(deadline
, now
))
1359 msecs
= jiffies_to_msecs(deadline
- now
);
1362 if (ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0,
1363 AHCI_CMD_RESET
| AHCI_CMD_CLR_BUSY
, msecs
)) {
1365 reason
= "1st FIS failed";
1369 /* spec says at least 5us, but be generous and sleep for 1ms */
1372 /* issue the second D2H Register FIS */
1373 tf
.ctl
&= ~ATA_SRST
;
1374 ahci_exec_polled_cmd(ap
, pmp
, &tf
, 0, 0, 0);
1376 /* wait for link to become ready */
1377 rc
= ata_wait_after_reset(link
, deadline
, check_ready
);
1378 if (rc
== -EBUSY
&& hpriv
->flags
& AHCI_HFLAG_SRST_TOUT_IS_OFFLINE
) {
1380 * Workaround for cases where link online status can't
1381 * be trusted. Treat device readiness timeout as link
1384 ata_link_info(link
, "device not ready, treating as offline\n");
1385 *class = ATA_DEV_NONE
;
1387 /* link occupied, -ENODEV too is an error */
1388 reason
= "device not ready";
1391 *class = ahci_dev_classify(ap
);
1393 /* re-enable FBS if disabled before */
1395 ahci_enable_fbs(ap
);
1397 DPRINTK("EXIT, class=%u\n", *class);
1401 ata_link_err(link
, "softreset failed (%s)\n", reason
);
1405 int ahci_check_ready(struct ata_link
*link
)
1407 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1408 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1410 return ata_check_ready(status
);
1412 EXPORT_SYMBOL_GPL(ahci_check_ready
);
1414 static int ahci_softreset(struct ata_link
*link
, unsigned int *class,
1415 unsigned long deadline
)
1417 int pmp
= sata_srst_pmp(link
);
1421 return ahci_do_softreset(link
, class, pmp
, deadline
, ahci_check_ready
);
1423 EXPORT_SYMBOL_GPL(ahci_do_softreset
);
1425 static int ahci_bad_pmp_check_ready(struct ata_link
*link
)
1427 void __iomem
*port_mmio
= ahci_port_base(link
->ap
);
1428 u8 status
= readl(port_mmio
+ PORT_TFDATA
) & 0xFF;
1429 u32 irq_status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1432 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1433 * which can save timeout delay.
1435 if (irq_status
& PORT_IRQ_BAD_PMP
)
1438 return ata_check_ready(status
);
1441 static int ahci_pmp_retry_softreset(struct ata_link
*link
, unsigned int *class,
1442 unsigned long deadline
)
1444 struct ata_port
*ap
= link
->ap
;
1445 void __iomem
*port_mmio
= ahci_port_base(ap
);
1446 int pmp
= sata_srst_pmp(link
);
1452 rc
= ahci_do_softreset(link
, class, pmp
, deadline
,
1453 ahci_bad_pmp_check_ready
);
1456 * Soft reset fails with IPMS set when PMP is enabled but
1457 * SATA HDD/ODD is connected to SATA port, do soft reset
1461 irq_sts
= readl(port_mmio
+ PORT_IRQ_STAT
);
1462 if (irq_sts
& PORT_IRQ_BAD_PMP
) {
1464 "applying PMP SRST workaround "
1466 rc
= ahci_do_softreset(link
, class, 0, deadline
,
1474 static int ahci_hardreset(struct ata_link
*link
, unsigned int *class,
1475 unsigned long deadline
)
1477 const unsigned long *timing
= sata_ehc_deb_timing(&link
->eh_context
);
1478 struct ata_port
*ap
= link
->ap
;
1479 struct ahci_port_priv
*pp
= ap
->private_data
;
1480 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1481 u8
*d2h_fis
= pp
->rx_fis
+ RX_FIS_D2H_REG
;
1482 struct ata_taskfile tf
;
1488 ahci_stop_engine(ap
);
1490 /* clear D2H reception area to properly wait for D2H FIS */
1491 ata_tf_init(link
->device
, &tf
);
1492 tf
.command
= ATA_BUSY
;
1493 ata_tf_to_fis(&tf
, 0, 0, d2h_fis
);
1495 rc
= sata_link_hardreset(link
, timing
, deadline
, &online
,
1498 hpriv
->start_engine(ap
);
1501 *class = ahci_dev_classify(ap
);
1503 DPRINTK("EXIT, rc=%d, class=%u\n", rc
, *class);
1507 static void ahci_postreset(struct ata_link
*link
, unsigned int *class)
1509 struct ata_port
*ap
= link
->ap
;
1510 void __iomem
*port_mmio
= ahci_port_base(ap
);
1513 ata_std_postreset(link
, class);
1515 /* Make sure port's ATAPI bit is set appropriately */
1516 new_tmp
= tmp
= readl(port_mmio
+ PORT_CMD
);
1517 if (*class == ATA_DEV_ATAPI
)
1518 new_tmp
|= PORT_CMD_ATAPI
;
1520 new_tmp
&= ~PORT_CMD_ATAPI
;
1521 if (new_tmp
!= tmp
) {
1522 writel(new_tmp
, port_mmio
+ PORT_CMD
);
1523 readl(port_mmio
+ PORT_CMD
); /* flush */
1527 static unsigned int ahci_fill_sg(struct ata_queued_cmd
*qc
, void *cmd_tbl
)
1529 struct scatterlist
*sg
;
1530 struct ahci_sg
*ahci_sg
= cmd_tbl
+ AHCI_CMD_TBL_HDR_SZ
;
1536 * Next, the S/G list.
1538 for_each_sg(qc
->sg
, sg
, qc
->n_elem
, si
) {
1539 dma_addr_t addr
= sg_dma_address(sg
);
1540 u32 sg_len
= sg_dma_len(sg
);
1542 ahci_sg
[si
].addr
= cpu_to_le32(addr
& 0xffffffff);
1543 ahci_sg
[si
].addr_hi
= cpu_to_le32((addr
>> 16) >> 16);
1544 ahci_sg
[si
].flags_size
= cpu_to_le32(sg_len
- 1);
1550 static int ahci_pmp_qc_defer(struct ata_queued_cmd
*qc
)
1552 struct ata_port
*ap
= qc
->ap
;
1553 struct ahci_port_priv
*pp
= ap
->private_data
;
1555 if (!sata_pmp_attached(ap
) || pp
->fbs_enabled
)
1556 return ata_std_qc_defer(qc
);
1558 return sata_pmp_qc_defer_cmd_switch(qc
);
1561 static void ahci_qc_prep(struct ata_queued_cmd
*qc
)
1563 struct ata_port
*ap
= qc
->ap
;
1564 struct ahci_port_priv
*pp
= ap
->private_data
;
1565 int is_atapi
= ata_is_atapi(qc
->tf
.protocol
);
1568 const u32 cmd_fis_len
= 5; /* five dwords */
1569 unsigned int n_elem
;
1572 * Fill in command table information. First, the header,
1573 * a SATA Register - Host to Device command FIS.
1575 cmd_tbl
= pp
->cmd_tbl
+ qc
->tag
* AHCI_CMD_TBL_SZ
;
1577 ata_tf_to_fis(&qc
->tf
, qc
->dev
->link
->pmp
, 1, cmd_tbl
);
1579 memset(cmd_tbl
+ AHCI_CMD_TBL_CDB
, 0, 32);
1580 memcpy(cmd_tbl
+ AHCI_CMD_TBL_CDB
, qc
->cdb
, qc
->dev
->cdb_len
);
1584 if (qc
->flags
& ATA_QCFLAG_DMAMAP
)
1585 n_elem
= ahci_fill_sg(qc
, cmd_tbl
);
1588 * Fill in command slot information.
1590 opts
= cmd_fis_len
| n_elem
<< 16 | (qc
->dev
->link
->pmp
<< 12);
1591 if (qc
->tf
.flags
& ATA_TFLAG_WRITE
)
1592 opts
|= AHCI_CMD_WRITE
;
1594 opts
|= AHCI_CMD_ATAPI
| AHCI_CMD_PREFETCH
;
1596 ahci_fill_cmd_slot(pp
, qc
->tag
, opts
);
1599 static void ahci_fbs_dec_intr(struct ata_port
*ap
)
1601 struct ahci_port_priv
*pp
= ap
->private_data
;
1602 void __iomem
*port_mmio
= ahci_port_base(ap
);
1603 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1607 BUG_ON(!pp
->fbs_enabled
);
1609 /* time to wait for DEC is not specified by AHCI spec,
1610 * add a retry loop for safety.
1612 writel(fbs
| PORT_FBS_DEC
, port_mmio
+ PORT_FBS
);
1613 fbs
= readl(port_mmio
+ PORT_FBS
);
1614 while ((fbs
& PORT_FBS_DEC
) && retries
--) {
1616 fbs
= readl(port_mmio
+ PORT_FBS
);
1619 if (fbs
& PORT_FBS_DEC
)
1620 dev_err(ap
->host
->dev
, "failed to clear device error\n");
1623 static void ahci_error_intr(struct ata_port
*ap
, u32 irq_stat
)
1625 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1626 struct ahci_port_priv
*pp
= ap
->private_data
;
1627 struct ata_eh_info
*host_ehi
= &ap
->link
.eh_info
;
1628 struct ata_link
*link
= NULL
;
1629 struct ata_queued_cmd
*active_qc
;
1630 struct ata_eh_info
*active_ehi
;
1631 bool fbs_need_dec
= false;
1634 /* determine active link with error */
1635 if (pp
->fbs_enabled
) {
1636 void __iomem
*port_mmio
= ahci_port_base(ap
);
1637 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1638 int pmp
= fbs
>> PORT_FBS_DWE_OFFSET
;
1640 if ((fbs
& PORT_FBS_SDE
) && (pmp
< ap
->nr_pmp_links
)) {
1641 link
= &ap
->pmp_link
[pmp
];
1642 fbs_need_dec
= true;
1646 ata_for_each_link(link
, ap
, EDGE
)
1647 if (ata_link_active(link
))
1653 active_qc
= ata_qc_from_tag(ap
, link
->active_tag
);
1654 active_ehi
= &link
->eh_info
;
1656 /* record irq stat */
1657 ata_ehi_clear_desc(host_ehi
);
1658 ata_ehi_push_desc(host_ehi
, "irq_stat 0x%08x", irq_stat
);
1660 /* AHCI needs SError cleared; otherwise, it might lock up */
1661 ahci_scr_read(&ap
->link
, SCR_ERROR
, &serror
);
1662 ahci_scr_write(&ap
->link
, SCR_ERROR
, serror
);
1663 host_ehi
->serror
|= serror
;
1665 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
1666 if (hpriv
->flags
& AHCI_HFLAG_IGN_IRQ_IF_ERR
)
1667 irq_stat
&= ~PORT_IRQ_IF_ERR
;
1669 if (irq_stat
& PORT_IRQ_TF_ERR
) {
1670 /* If qc is active, charge it; otherwise, the active
1671 * link. There's no active qc on NCQ errors. It will
1672 * be determined by EH by reading log page 10h.
1675 active_qc
->err_mask
|= AC_ERR_DEV
;
1677 active_ehi
->err_mask
|= AC_ERR_DEV
;
1679 if (hpriv
->flags
& AHCI_HFLAG_IGN_SERR_INTERNAL
)
1680 host_ehi
->serror
&= ~SERR_INTERNAL
;
1683 if (irq_stat
& PORT_IRQ_UNK_FIS
) {
1684 u32
*unk
= pp
->rx_fis
+ RX_FIS_UNK
;
1686 active_ehi
->err_mask
|= AC_ERR_HSM
;
1687 active_ehi
->action
|= ATA_EH_RESET
;
1688 ata_ehi_push_desc(active_ehi
,
1689 "unknown FIS %08x %08x %08x %08x" ,
1690 unk
[0], unk
[1], unk
[2], unk
[3]);
1693 if (sata_pmp_attached(ap
) && (irq_stat
& PORT_IRQ_BAD_PMP
)) {
1694 active_ehi
->err_mask
|= AC_ERR_HSM
;
1695 active_ehi
->action
|= ATA_EH_RESET
;
1696 ata_ehi_push_desc(active_ehi
, "incorrect PMP");
1699 if (irq_stat
& (PORT_IRQ_HBUS_ERR
| PORT_IRQ_HBUS_DATA_ERR
)) {
1700 host_ehi
->err_mask
|= AC_ERR_HOST_BUS
;
1701 host_ehi
->action
|= ATA_EH_RESET
;
1702 ata_ehi_push_desc(host_ehi
, "host bus error");
1705 if (irq_stat
& PORT_IRQ_IF_ERR
) {
1707 active_ehi
->err_mask
|= AC_ERR_DEV
;
1709 host_ehi
->err_mask
|= AC_ERR_ATA_BUS
;
1710 host_ehi
->action
|= ATA_EH_RESET
;
1713 ata_ehi_push_desc(host_ehi
, "interface fatal error");
1716 if (irq_stat
& (PORT_IRQ_CONNECT
| PORT_IRQ_PHYRDY
)) {
1717 ata_ehi_hotplugged(host_ehi
);
1718 ata_ehi_push_desc(host_ehi
, "%s",
1719 irq_stat
& PORT_IRQ_CONNECT
?
1720 "connection status changed" : "PHY RDY changed");
1723 /* okay, let's hand over to EH */
1725 if (irq_stat
& PORT_IRQ_FREEZE
)
1726 ata_port_freeze(ap
);
1727 else if (fbs_need_dec
) {
1728 ata_link_abort(link
);
1729 ahci_fbs_dec_intr(ap
);
1734 static void ahci_handle_port_interrupt(struct ata_port
*ap
,
1735 void __iomem
*port_mmio
, u32 status
)
1737 struct ata_eh_info
*ehi
= &ap
->link
.eh_info
;
1738 struct ahci_port_priv
*pp
= ap
->private_data
;
1739 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
1740 int resetting
= !!(ap
->pflags
& ATA_PFLAG_RESETTING
);
1744 /* ignore BAD_PMP while resetting */
1745 if (unlikely(resetting
))
1746 status
&= ~PORT_IRQ_BAD_PMP
;
1748 if (sata_lpm_ignore_phy_events(&ap
->link
)) {
1749 status
&= ~PORT_IRQ_PHYRDY
;
1750 ahci_scr_write(&ap
->link
, SCR_ERROR
, SERR_PHYRDY_CHG
);
1753 if (unlikely(status
& PORT_IRQ_ERROR
)) {
1754 ahci_error_intr(ap
, status
);
1758 if (status
& PORT_IRQ_SDB_FIS
) {
1759 /* If SNotification is available, leave notification
1760 * handling to sata_async_notification(). If not,
1761 * emulate it by snooping SDB FIS RX area.
1763 * Snooping FIS RX area is probably cheaper than
1764 * poking SNotification but some constrollers which
1765 * implement SNotification, ICH9 for example, don't
1766 * store AN SDB FIS into receive area.
1768 if (hpriv
->cap
& HOST_CAP_SNTF
)
1769 sata_async_notification(ap
);
1771 /* If the 'N' bit in word 0 of the FIS is set,
1772 * we just received asynchronous notification.
1773 * Tell libata about it.
1775 * Lack of SNotification should not appear in
1776 * ahci 1.2, so the workaround is unnecessary
1777 * when FBS is enabled.
1779 if (pp
->fbs_enabled
)
1782 const __le32
*f
= pp
->rx_fis
+ RX_FIS_SDB
;
1783 u32 f0
= le32_to_cpu(f
[0]);
1785 sata_async_notification(ap
);
1790 /* pp->active_link is not reliable once FBS is enabled, both
1791 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1792 * NCQ and non-NCQ commands may be in flight at the same time.
1794 if (pp
->fbs_enabled
) {
1795 if (ap
->qc_active
) {
1796 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1797 qc_active
|= readl(port_mmio
+ PORT_CMD_ISSUE
);
1800 /* pp->active_link is valid iff any command is in flight */
1801 if (ap
->qc_active
&& pp
->active_link
->sactive
)
1802 qc_active
= readl(port_mmio
+ PORT_SCR_ACT
);
1804 qc_active
= readl(port_mmio
+ PORT_CMD_ISSUE
);
1808 rc
= ata_qc_complete_multiple(ap
, qc_active
);
1810 /* while resetting, invalid completions are expected */
1811 if (unlikely(rc
< 0 && !resetting
)) {
1812 ehi
->err_mask
|= AC_ERR_HSM
;
1813 ehi
->action
|= ATA_EH_RESET
;
1814 ata_port_freeze(ap
);
1818 static void ahci_port_intr(struct ata_port
*ap
)
1820 void __iomem
*port_mmio
= ahci_port_base(ap
);
1823 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1824 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1826 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1829 static irqreturn_t
ahci_multi_irqs_intr_hard(int irq
, void *dev_instance
)
1831 struct ata_port
*ap
= dev_instance
;
1832 void __iomem
*port_mmio
= ahci_port_base(ap
);
1837 status
= readl(port_mmio
+ PORT_IRQ_STAT
);
1838 writel(status
, port_mmio
+ PORT_IRQ_STAT
);
1840 spin_lock(ap
->lock
);
1841 ahci_handle_port_interrupt(ap
, port_mmio
, status
);
1842 spin_unlock(ap
->lock
);
1849 static u32
ahci_handle_port_intr(struct ata_host
*host
, u32 irq_masked
)
1851 unsigned int i
, handled
= 0;
1853 for (i
= 0; i
< host
->n_ports
; i
++) {
1854 struct ata_port
*ap
;
1856 if (!(irq_masked
& (1 << i
)))
1859 ap
= host
->ports
[i
];
1862 VPRINTK("port %u\n", i
);
1864 VPRINTK("port %u (no irq)\n", i
);
1865 if (ata_ratelimit())
1867 "interrupt on disabled port %u\n", i
);
1876 static irqreturn_t
ahci_single_edge_irq_intr(int irq
, void *dev_instance
)
1878 struct ata_host
*host
= dev_instance
;
1879 struct ahci_host_priv
*hpriv
;
1880 unsigned int rc
= 0;
1882 u32 irq_stat
, irq_masked
;
1886 hpriv
= host
->private_data
;
1889 /* sigh. 0xffffffff is a valid return from h/w */
1890 irq_stat
= readl(mmio
+ HOST_IRQ_STAT
);
1894 irq_masked
= irq_stat
& hpriv
->port_map
;
1896 spin_lock(&host
->lock
);
1899 * HOST_IRQ_STAT behaves as edge triggered latch meaning that
1900 * it should be cleared before all the port events are cleared.
1902 writel(irq_stat
, mmio
+ HOST_IRQ_STAT
);
1904 rc
= ahci_handle_port_intr(host
, irq_masked
);
1906 spin_unlock(&host
->lock
);
1910 return IRQ_RETVAL(rc
);
1913 static irqreturn_t
ahci_single_level_irq_intr(int irq
, void *dev_instance
)
1915 struct ata_host
*host
= dev_instance
;
1916 struct ahci_host_priv
*hpriv
;
1917 unsigned int rc
= 0;
1919 u32 irq_stat
, irq_masked
;
1923 hpriv
= host
->private_data
;
1926 /* sigh. 0xffffffff is a valid return from h/w */
1927 irq_stat
= readl(mmio
+ HOST_IRQ_STAT
);
1931 irq_masked
= irq_stat
& hpriv
->port_map
;
1933 spin_lock(&host
->lock
);
1935 rc
= ahci_handle_port_intr(host
, irq_masked
);
1937 /* HOST_IRQ_STAT behaves as level triggered latch meaning that
1938 * it should be cleared after all the port events are cleared;
1939 * otherwise, it will raise a spurious interrupt after each
1940 * valid one. Please read section 10.6.2 of ahci 1.1 for more
1943 * Also, use the unmasked value to clear interrupt as spurious
1944 * pending event on a dummy port might cause screaming IRQ.
1946 writel(irq_stat
, mmio
+ HOST_IRQ_STAT
);
1948 spin_unlock(&host
->lock
);
1952 return IRQ_RETVAL(rc
);
1955 unsigned int ahci_qc_issue(struct ata_queued_cmd
*qc
)
1957 struct ata_port
*ap
= qc
->ap
;
1958 void __iomem
*port_mmio
= ahci_port_base(ap
);
1959 struct ahci_port_priv
*pp
= ap
->private_data
;
1961 /* Keep track of the currently active link. It will be used
1962 * in completion path to determine whether NCQ phase is in
1965 pp
->active_link
= qc
->dev
->link
;
1967 if (qc
->tf
.protocol
== ATA_PROT_NCQ
)
1968 writel(1 << qc
->tag
, port_mmio
+ PORT_SCR_ACT
);
1970 if (pp
->fbs_enabled
&& pp
->fbs_last_dev
!= qc
->dev
->link
->pmp
) {
1971 u32 fbs
= readl(port_mmio
+ PORT_FBS
);
1972 fbs
&= ~(PORT_FBS_DEV_MASK
| PORT_FBS_DEC
);
1973 fbs
|= qc
->dev
->link
->pmp
<< PORT_FBS_DEV_OFFSET
;
1974 writel(fbs
, port_mmio
+ PORT_FBS
);
1975 pp
->fbs_last_dev
= qc
->dev
->link
->pmp
;
1978 writel(1 << qc
->tag
, port_mmio
+ PORT_CMD_ISSUE
);
1980 ahci_sw_activity(qc
->dev
->link
);
1984 EXPORT_SYMBOL_GPL(ahci_qc_issue
);
1986 static bool ahci_qc_fill_rtf(struct ata_queued_cmd
*qc
)
1988 struct ahci_port_priv
*pp
= qc
->ap
->private_data
;
1989 u8
*rx_fis
= pp
->rx_fis
;
1991 if (pp
->fbs_enabled
)
1992 rx_fis
+= qc
->dev
->link
->pmp
* AHCI_RX_FIS_SZ
;
1995 * After a successful execution of an ATA PIO data-in command,
1996 * the device doesn't send D2H Reg FIS to update the TF and
1997 * the host should take TF and E_Status from the preceding PIO
2000 if (qc
->tf
.protocol
== ATA_PROT_PIO
&& qc
->dma_dir
== DMA_FROM_DEVICE
&&
2001 !(qc
->flags
& ATA_QCFLAG_FAILED
)) {
2002 ata_tf_from_fis(rx_fis
+ RX_FIS_PIO_SETUP
, &qc
->result_tf
);
2003 qc
->result_tf
.command
= (rx_fis
+ RX_FIS_PIO_SETUP
)[15];
2005 ata_tf_from_fis(rx_fis
+ RX_FIS_D2H_REG
, &qc
->result_tf
);
2010 static void ahci_freeze(struct ata_port
*ap
)
2012 void __iomem
*port_mmio
= ahci_port_base(ap
);
2015 writel(0, port_mmio
+ PORT_IRQ_MASK
);
2018 static void ahci_thaw(struct ata_port
*ap
)
2020 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2021 void __iomem
*mmio
= hpriv
->mmio
;
2022 void __iomem
*port_mmio
= ahci_port_base(ap
);
2024 struct ahci_port_priv
*pp
= ap
->private_data
;
2027 tmp
= readl(port_mmio
+ PORT_IRQ_STAT
);
2028 writel(tmp
, port_mmio
+ PORT_IRQ_STAT
);
2029 writel(1 << ap
->port_no
, mmio
+ HOST_IRQ_STAT
);
2031 /* turn IRQ back on */
2032 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2035 void ahci_error_handler(struct ata_port
*ap
)
2037 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2039 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
)) {
2040 /* restart engine */
2041 ahci_stop_engine(ap
);
2042 hpriv
->start_engine(ap
);
2045 sata_pmp_error_handler(ap
);
2047 if (!ata_dev_enabled(ap
->link
.device
))
2048 ahci_stop_engine(ap
);
2050 EXPORT_SYMBOL_GPL(ahci_error_handler
);
2052 static void ahci_post_internal_cmd(struct ata_queued_cmd
*qc
)
2054 struct ata_port
*ap
= qc
->ap
;
2056 /* make DMA engine forget about the failed command */
2057 if (qc
->flags
& ATA_QCFLAG_FAILED
)
2058 ahci_kick_engine(ap
);
2061 static void ahci_set_aggressive_devslp(struct ata_port
*ap
, bool sleep
)
2063 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2064 void __iomem
*port_mmio
= ahci_port_base(ap
);
2065 struct ata_device
*dev
= ap
->link
.device
;
2066 u32 devslp
, dm
, dito
, mdat
, deto
;
2068 unsigned int err_mask
;
2070 devslp
= readl(port_mmio
+ PORT_DEVSLP
);
2071 if (!(devslp
& PORT_DEVSLP_DSP
)) {
2072 dev_info(ap
->host
->dev
, "port does not support device sleep\n");
2076 /* disable device sleep */
2078 if (devslp
& PORT_DEVSLP_ADSE
) {
2079 writel(devslp
& ~PORT_DEVSLP_ADSE
,
2080 port_mmio
+ PORT_DEVSLP
);
2081 err_mask
= ata_dev_set_feature(dev
,
2082 SETFEATURES_SATA_DISABLE
,
2084 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2085 ata_dev_warn(dev
, "failed to disable DEVSLP\n");
2090 /* device sleep was already enabled */
2091 if (devslp
& PORT_DEVSLP_ADSE
)
2094 /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2095 rc
= ahci_stop_engine(ap
);
2099 dm
= (devslp
& PORT_DEVSLP_DM_MASK
) >> PORT_DEVSLP_DM_OFFSET
;
2100 dito
= devslp_idle_timeout
/ (dm
+ 1);
2104 /* Use the nominal value 10 ms if the read MDAT is zero,
2105 * the nominal value of DETO is 20 ms.
2107 if (dev
->devslp_timing
[ATA_LOG_DEVSLP_VALID
] &
2108 ATA_LOG_DEVSLP_VALID_MASK
) {
2109 mdat
= dev
->devslp_timing
[ATA_LOG_DEVSLP_MDAT
] &
2110 ATA_LOG_DEVSLP_MDAT_MASK
;
2113 deto
= dev
->devslp_timing
[ATA_LOG_DEVSLP_DETO
];
2121 devslp
|= ((dito
<< PORT_DEVSLP_DITO_OFFSET
) |
2122 (mdat
<< PORT_DEVSLP_MDAT_OFFSET
) |
2123 (deto
<< PORT_DEVSLP_DETO_OFFSET
) |
2125 writel(devslp
, port_mmio
+ PORT_DEVSLP
);
2127 hpriv
->start_engine(ap
);
2129 /* enable device sleep feature for the drive */
2130 err_mask
= ata_dev_set_feature(dev
,
2131 SETFEATURES_SATA_ENABLE
,
2133 if (err_mask
&& err_mask
!= AC_ERR_DEV
)
2134 ata_dev_warn(dev
, "failed to enable DEVSLP\n");
2137 static void ahci_enable_fbs(struct ata_port
*ap
)
2139 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2140 struct ahci_port_priv
*pp
= ap
->private_data
;
2141 void __iomem
*port_mmio
= ahci_port_base(ap
);
2145 if (!pp
->fbs_supported
)
2148 fbs
= readl(port_mmio
+ PORT_FBS
);
2149 if (fbs
& PORT_FBS_EN
) {
2150 pp
->fbs_enabled
= true;
2151 pp
->fbs_last_dev
= -1; /* initialization */
2155 rc
= ahci_stop_engine(ap
);
2159 writel(fbs
| PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2160 fbs
= readl(port_mmio
+ PORT_FBS
);
2161 if (fbs
& PORT_FBS_EN
) {
2162 dev_info(ap
->host
->dev
, "FBS is enabled\n");
2163 pp
->fbs_enabled
= true;
2164 pp
->fbs_last_dev
= -1; /* initialization */
2166 dev_err(ap
->host
->dev
, "Failed to enable FBS\n");
2168 hpriv
->start_engine(ap
);
2171 static void ahci_disable_fbs(struct ata_port
*ap
)
2173 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2174 struct ahci_port_priv
*pp
= ap
->private_data
;
2175 void __iomem
*port_mmio
= ahci_port_base(ap
);
2179 if (!pp
->fbs_supported
)
2182 fbs
= readl(port_mmio
+ PORT_FBS
);
2183 if ((fbs
& PORT_FBS_EN
) == 0) {
2184 pp
->fbs_enabled
= false;
2188 rc
= ahci_stop_engine(ap
);
2192 writel(fbs
& ~PORT_FBS_EN
, port_mmio
+ PORT_FBS
);
2193 fbs
= readl(port_mmio
+ PORT_FBS
);
2194 if (fbs
& PORT_FBS_EN
)
2195 dev_err(ap
->host
->dev
, "Failed to disable FBS\n");
2197 dev_info(ap
->host
->dev
, "FBS is disabled\n");
2198 pp
->fbs_enabled
= false;
2201 hpriv
->start_engine(ap
);
2204 static void ahci_pmp_attach(struct ata_port
*ap
)
2206 void __iomem
*port_mmio
= ahci_port_base(ap
);
2207 struct ahci_port_priv
*pp
= ap
->private_data
;
2210 cmd
= readl(port_mmio
+ PORT_CMD
);
2211 cmd
|= PORT_CMD_PMP
;
2212 writel(cmd
, port_mmio
+ PORT_CMD
);
2214 ahci_enable_fbs(ap
);
2216 pp
->intr_mask
|= PORT_IRQ_BAD_PMP
;
2219 * We must not change the port interrupt mask register if the
2220 * port is marked frozen, the value in pp->intr_mask will be
2221 * restored later when the port is thawed.
2223 * Note that during initialization, the port is marked as
2224 * frozen since the irq handler is not yet registered.
2226 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2227 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2230 static void ahci_pmp_detach(struct ata_port
*ap
)
2232 void __iomem
*port_mmio
= ahci_port_base(ap
);
2233 struct ahci_port_priv
*pp
= ap
->private_data
;
2236 ahci_disable_fbs(ap
);
2238 cmd
= readl(port_mmio
+ PORT_CMD
);
2239 cmd
&= ~PORT_CMD_PMP
;
2240 writel(cmd
, port_mmio
+ PORT_CMD
);
2242 pp
->intr_mask
&= ~PORT_IRQ_BAD_PMP
;
2244 /* see comment above in ahci_pmp_attach() */
2245 if (!(ap
->pflags
& ATA_PFLAG_FROZEN
))
2246 writel(pp
->intr_mask
, port_mmio
+ PORT_IRQ_MASK
);
2249 int ahci_port_resume(struct ata_port
*ap
)
2252 ahci_start_port(ap
);
2254 if (sata_pmp_attached(ap
))
2255 ahci_pmp_attach(ap
);
2257 ahci_pmp_detach(ap
);
2261 EXPORT_SYMBOL_GPL(ahci_port_resume
);
2264 static int ahci_port_suspend(struct ata_port
*ap
, pm_message_t mesg
)
2266 const char *emsg
= NULL
;
2269 rc
= ahci_deinit_port(ap
, &emsg
);
2271 ahci_power_down(ap
);
2273 ata_port_err(ap
, "%s (%d)\n", emsg
, rc
);
2274 ata_port_freeze(ap
);
2281 static int ahci_port_start(struct ata_port
*ap
)
2283 struct ahci_host_priv
*hpriv
= ap
->host
->private_data
;
2284 struct device
*dev
= ap
->host
->dev
;
2285 struct ahci_port_priv
*pp
;
2288 size_t dma_sz
, rx_fis_sz
;
2290 pp
= devm_kzalloc(dev
, sizeof(*pp
), GFP_KERNEL
);
2294 if (ap
->host
->n_ports
> 1) {
2295 pp
->irq_desc
= devm_kzalloc(dev
, 8, GFP_KERNEL
);
2296 if (!pp
->irq_desc
) {
2297 devm_kfree(dev
, pp
);
2300 snprintf(pp
->irq_desc
, 8,
2301 "%s%d", dev_driver_string(dev
), ap
->port_no
);
2304 /* check FBS capability */
2305 if ((hpriv
->cap
& HOST_CAP_FBS
) && sata_pmp_supported(ap
)) {
2306 void __iomem
*port_mmio
= ahci_port_base(ap
);
2307 u32 cmd
= readl(port_mmio
+ PORT_CMD
);
2308 if (cmd
& PORT_CMD_FBSCP
)
2309 pp
->fbs_supported
= true;
2310 else if (hpriv
->flags
& AHCI_HFLAG_YES_FBS
) {
2311 dev_info(dev
, "port %d can do FBS, forcing FBSCP\n",
2313 pp
->fbs_supported
= true;
2315 dev_warn(dev
, "port %d is not capable of FBS\n",
2319 if (pp
->fbs_supported
) {
2320 dma_sz
= AHCI_PORT_PRIV_FBS_DMA_SZ
;
2321 rx_fis_sz
= AHCI_RX_FIS_SZ
* 16;
2323 dma_sz
= AHCI_PORT_PRIV_DMA_SZ
;
2324 rx_fis_sz
= AHCI_RX_FIS_SZ
;
2327 mem
= dmam_alloc_coherent(dev
, dma_sz
, &mem_dma
, GFP_KERNEL
);
2330 memset(mem
, 0, dma_sz
);
2333 * First item in chunk of DMA memory: 32-slot command table,
2334 * 32 bytes each in size
2337 pp
->cmd_slot_dma
= mem_dma
;
2339 mem
+= AHCI_CMD_SLOT_SZ
;
2340 mem_dma
+= AHCI_CMD_SLOT_SZ
;
2343 * Second item: Received-FIS area
2346 pp
->rx_fis_dma
= mem_dma
;
2349 mem_dma
+= rx_fis_sz
;
2352 * Third item: data area for storing a single command
2353 * and its scatter-gather table
2356 pp
->cmd_tbl_dma
= mem_dma
;
2359 * Save off initial list of interrupts to be enabled.
2360 * This could be changed later
2362 pp
->intr_mask
= DEF_PORT_IRQ
;
2365 * Switch to per-port locking in case each port has its own MSI vector.
2367 if (hpriv
->flags
& AHCI_HFLAG_MULTI_MSI
) {
2368 spin_lock_init(&pp
->lock
);
2369 ap
->lock
= &pp
->lock
;
2372 ap
->private_data
= pp
;
2374 /* engage engines, captain */
2375 return ahci_port_resume(ap
);
2378 static void ahci_port_stop(struct ata_port
*ap
)
2380 const char *emsg
= NULL
;
2383 /* de-initialize port */
2384 rc
= ahci_deinit_port(ap
, &emsg
);
2386 ata_port_warn(ap
, "%s (%d)\n", emsg
, rc
);
2389 void ahci_print_info(struct ata_host
*host
, const char *scc_s
)
2391 struct ahci_host_priv
*hpriv
= host
->private_data
;
2392 void __iomem
*mmio
= hpriv
->mmio
;
2393 u32 vers
, cap
, cap2
, impl
, speed
;
2394 const char *speed_s
;
2396 vers
= readl(mmio
+ HOST_VERSION
);
2399 impl
= hpriv
->port_map
;
2401 speed
= (cap
>> 20) & 0xf;
2404 else if (speed
== 2)
2406 else if (speed
== 3)
2412 "AHCI %02x%02x.%02x%02x "
2413 "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2416 (vers
>> 24) & 0xff,
2417 (vers
>> 16) & 0xff,
2421 ((cap
>> 8) & 0x1f) + 1,
2435 cap
& HOST_CAP_64
? "64bit " : "",
2436 cap
& HOST_CAP_NCQ
? "ncq " : "",
2437 cap
& HOST_CAP_SNTF
? "sntf " : "",
2438 cap
& HOST_CAP_MPS
? "ilck " : "",
2439 cap
& HOST_CAP_SSS
? "stag " : "",
2440 cap
& HOST_CAP_ALPM
? "pm " : "",
2441 cap
& HOST_CAP_LED
? "led " : "",
2442 cap
& HOST_CAP_CLO
? "clo " : "",
2443 cap
& HOST_CAP_ONLY
? "only " : "",
2444 cap
& HOST_CAP_PMP
? "pmp " : "",
2445 cap
& HOST_CAP_FBS
? "fbs " : "",
2446 cap
& HOST_CAP_PIO_MULTI
? "pio " : "",
2447 cap
& HOST_CAP_SSC
? "slum " : "",
2448 cap
& HOST_CAP_PART
? "part " : "",
2449 cap
& HOST_CAP_CCC
? "ccc " : "",
2450 cap
& HOST_CAP_EMS
? "ems " : "",
2451 cap
& HOST_CAP_SXS
? "sxs " : "",
2452 cap2
& HOST_CAP2_DESO
? "deso " : "",
2453 cap2
& HOST_CAP2_SADM
? "sadm " : "",
2454 cap2
& HOST_CAP2_SDS
? "sds " : "",
2455 cap2
& HOST_CAP2_APST
? "apst " : "",
2456 cap2
& HOST_CAP2_NVMHCI
? "nvmp " : "",
2457 cap2
& HOST_CAP2_BOH
? "boh " : ""
2460 EXPORT_SYMBOL_GPL(ahci_print_info
);
2462 void ahci_set_em_messages(struct ahci_host_priv
*hpriv
,
2463 struct ata_port_info
*pi
)
2466 void __iomem
*mmio
= hpriv
->mmio
;
2467 u32 em_loc
= readl(mmio
+ HOST_EM_LOC
);
2468 u32 em_ctl
= readl(mmio
+ HOST_EM_CTL
);
2470 if (!ahci_em_messages
|| !(hpriv
->cap
& HOST_CAP_EMS
))
2473 messages
= (em_ctl
& EM_CTRL_MSG_TYPE
) >> 16;
2477 hpriv
->em_loc
= ((em_loc
>> 16) * 4);
2478 hpriv
->em_buf_sz
= ((em_loc
& 0xff) * 4);
2479 hpriv
->em_msg_type
= messages
;
2480 pi
->flags
|= ATA_FLAG_EM
;
2481 if (!(em_ctl
& EM_CTL_ALHD
))
2482 pi
->flags
|= ATA_FLAG_SW_ACTIVITY
;
2485 EXPORT_SYMBOL_GPL(ahci_set_em_messages
);
2487 static int ahci_host_activate_multi_irqs(struct ata_host
*host
,
2488 struct scsi_host_template
*sht
)
2490 struct ahci_host_priv
*hpriv
= host
->private_data
;
2493 rc
= ata_host_start(host
);
2497 * Requests IRQs according to AHCI-1.1 when multiple MSIs were
2498 * allocated. That is one MSI per port, starting from @irq.
2500 for (i
= 0; i
< host
->n_ports
; i
++) {
2501 struct ahci_port_priv
*pp
= host
->ports
[i
]->private_data
;
2502 int irq
= ahci_irq_vector(hpriv
, i
);
2504 /* Do not receive interrupts sent by dummy ports */
2506 disable_irq(irq
+ i
);
2510 rc
= devm_request_irq(host
->dev
, irq
, ahci_multi_irqs_intr_hard
,
2511 0, pp
->irq_desc
, host
->ports
[i
]);
2515 ata_port_desc(host
->ports
[i
], "irq %d", irq
);
2518 return ata_host_register(host
, sht
);
2522 * ahci_host_activate - start AHCI host, request IRQs and register it
2523 * @host: target ATA host
2524 * @sht: scsi_host_template to use when registering the host
2527 * Inherited from calling layer (may sleep).
2530 * 0 on success, -errno otherwise.
2532 int ahci_host_activate(struct ata_host
*host
, struct scsi_host_template
*sht
)
2534 struct ahci_host_priv
*hpriv
= host
->private_data
;
2535 int irq
= hpriv
->irq
;
2538 if (hpriv
->flags
& (AHCI_HFLAG_MULTI_MSI
| AHCI_HFLAG_MULTI_MSIX
))
2539 rc
= ahci_host_activate_multi_irqs(host
, sht
);
2540 else if (hpriv
->flags
& AHCI_HFLAG_EDGE_IRQ
)
2541 rc
= ata_host_activate(host
, irq
, ahci_single_edge_irq_intr
,
2544 rc
= ata_host_activate(host
, irq
, ahci_single_level_irq_intr
,
2548 EXPORT_SYMBOL_GPL(ahci_host_activate
);
2550 MODULE_AUTHOR("Jeff Garzik");
2551 MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2552 MODULE_LICENSE("GPL");