]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/ata/sata_nv.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libata: change ATA_QCFLAG_DMAMAP semantics
[mirror_ubuntu-artful-kernel.git] / drivers / ata / sata_nv.c
CommitLineData
1da177e4
LT		 1/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
aa7e16d6
JG		 7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
1da177e4 21 *
af36d7f0
JG		 22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
fbbb262d
RH		 32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
1da177e4
LT		 37 */
38
1da177e4
LT		 39#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
a9524a76 46#include <linux/device.h>
1da177e4 47#include <scsi/scsi_host.h>
fbbb262d 48#include <scsi/scsi_device.h>
1da177e4
LT		 49#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
2a3103ce 52#define DRV_VERSION "3.5"
fbbb262d
RH		 53
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 55
10ad05df 56enum {
0d5ff566
TH		 57 NV_MMIO_BAR = 5,
58
10ad05df
JG		 59 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 65
27e4b274 66 /* INT_STATUS/ENABLE */
10ad05df 67 NV_INT_STATUS = 0x10,
10ad05df 68 NV_INT_ENABLE = 0x11,
27e4b274 69 NV_INT_STATUS_CK804 = 0x440,
10ad05df 70 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 71
27e4b274
TH		 72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
39f87582 80 NV_INT_ALL = 0x0f,
5a44efff
TH		 81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 83
27e4b274 84 /* INT_CONFIG */
10ad05df
JG		 85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 87
10ad05df
JG		 88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 166 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 171
f140f0f1
KL		 172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
fbbb262d
RH		 201};
202
203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
2dcb407e 231 u8 len; /* 3 */
fbbb262d
RH		 232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
10ad05df 239};
1da177e4 240
fbbb262d
RH		 241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
2dcb407e
JG		 247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
fbbb262d 250 u8 flags;
5e5c74a5 251 int last_issue_ncq;
fbbb262d
RH		 252};
253
cdf56bcf
RH		 254struct nv_host_priv {
255 unsigned long type;
256};
257
f140f0f1
KL		 258struct defer_queue {
259 u32 defer_bits;
260 unsigned int head;
261 unsigned int tail;
262 unsigned int tag[ATA_MAX_QUEUE];
263};
264
265enum ncq_saw_flag_list {
266 ncq_saw_d2h = (1U << 0),
267 ncq_saw_dmas = (1U << 1),
268 ncq_saw_sdb = (1U << 2),
269 ncq_saw_backout = (1U << 3),
270};
271
272struct nv_swncq_port_priv {
273 struct ata_prd *prd; /* our SG list */
274 dma_addr_t prd_dma; /* and its DMA mapping */
275 void __iomem *sactive_block;
276 void __iomem *irq_block;
277 void __iomem *tag_block;
278 u32 qc_active;
279
280 unsigned int last_issue_tag;
281
282 /* fifo circular queue to store deferral command */
283 struct defer_queue defer_queue;
284
285 /* for NCQ interrupt analysis */
286 u32 dhfis_bits;
287 u32 dmafis_bits;
288 u32 sdbfis_bits;
289
290 unsigned int ncq_flags;
291};
292
293
5796d1c4 294#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 295
2dcb407e 296static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
438ac6d5 297#ifdef CONFIG_PM
cdf56bcf 298static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 299#endif
cca3974e 300static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 301static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
302static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
2dcb407e
JG		 304static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
305static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
1da177e4 306
39f87582
TH		 307static void nv_nf2_freeze(struct ata_port *ap);
308static void nv_nf2_thaw(struct ata_port *ap);
309static void nv_ck804_freeze(struct ata_port *ap);
310static void nv_ck804_thaw(struct ata_port *ap);
311static void nv_error_handler(struct ata_port *ap);
fbbb262d 312static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 313static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
fbbb262d
RH		 314static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
315static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
316static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
317static void nv_adma_irq_clear(struct ata_port *ap);
318static int nv_adma_port_start(struct ata_port *ap);
319static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 320#ifdef CONFIG_PM
cdf56bcf
RH		 321static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
322static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 323#endif
53014e25
RH		 324static void nv_adma_freeze(struct ata_port *ap);
325static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 326static void nv_adma_error_handler(struct ata_port *ap);
327static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 328static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 329static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 330
f140f0f1
KL		 331static void nv_mcp55_thaw(struct ata_port *ap);
332static void nv_mcp55_freeze(struct ata_port *ap);
333static void nv_swncq_error_handler(struct ata_port *ap);
334static int nv_swncq_slave_config(struct scsi_device *sdev);
335static int nv_swncq_port_start(struct ata_port *ap);
336static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
337static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
338static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
339static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
340static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
341#ifdef CONFIG_PM
342static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
343static int nv_swncq_port_resume(struct ata_port *ap);
344#endif
345
1da177e4
LT		 346enum nv_host_type
347{
348 GENERIC,
349 NFORCE2,
27e4b274 350 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 351 CK804,
f140f0f1
KL		 352 ADMA,
353 SWNCQ,
1da177e4
LT		 354};
355
3b7d697d 356static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 357 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
359 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
f140f0f1
KL		 364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
e2e031eb
KL		 368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 371
372 { } /* terminate list */
1da177e4
LT		 373};
374
1da177e4
LT		 375static struct pci_driver nv_pci_driver = {
376 .name = DRV_NAME,
377 .id_table = nv_pci_tbl,
378 .probe = nv_init_one,
438ac6d5 379#ifdef CONFIG_PM
cdf56bcf
RH		 380 .suspend = ata_pci_device_suspend,
381 .resume = nv_pci_device_resume,
438ac6d5 382#endif
1daf9ce7 383 .remove = ata_pci_remove_one,
1da177e4
LT		 384};
385
193515d5 386static struct scsi_host_template nv_sht = {
1da177e4
LT		 387 .module = THIS_MODULE,
388 .name = DRV_NAME,
389 .ioctl = ata_scsi_ioctl,
390 .queuecommand = ata_scsi_queuecmd,
1da177e4
LT		 391 .can_queue = ATA_DEF_QUEUE,
392 .this_id = ATA_SHT_THIS_ID,
393 .sg_tablesize = LIBATA_MAX_PRD,
1da177e4
LT		 394 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
395 .emulated = ATA_SHT_EMULATED,
396 .use_clustering = ATA_SHT_USE_CLUSTERING,
397 .proc_name = DRV_NAME,
398 .dma_boundary = ATA_DMA_BOUNDARY,
399 .slave_configure = ata_scsi_slave_config,
ccf68c34 400 .slave_destroy = ata_scsi_slave_destroy,
1da177e4 401 .bios_param = ata_std_bios_param,
1da177e4
LT		 402};
403
fbbb262d
RH		 404static struct scsi_host_template nv_adma_sht = {
405 .module = THIS_MODULE,
406 .name = DRV_NAME,
407 .ioctl = ata_scsi_ioctl,
408 .queuecommand = ata_scsi_queuecmd,
1e0b5ab8 409 .change_queue_depth = ata_scsi_change_queue_depth,
fbbb262d
RH		 410 .can_queue = NV_ADMA_MAX_CPBS,
411 .this_id = ATA_SHT_THIS_ID,
412 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 413 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
414 .emulated = ATA_SHT_EMULATED,
415 .use_clustering = ATA_SHT_USE_CLUSTERING,
416 .proc_name = DRV_NAME,
417 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
418 .slave_configure = nv_adma_slave_config,
419 .slave_destroy = ata_scsi_slave_destroy,
420 .bios_param = ata_std_bios_param,
421};
422
f140f0f1
KL		 423static struct scsi_host_template nv_swncq_sht = {
424 .module = THIS_MODULE,
425 .name = DRV_NAME,
426 .ioctl = ata_scsi_ioctl,
427 .queuecommand = ata_scsi_queuecmd,
428 .change_queue_depth = ata_scsi_change_queue_depth,
429 .can_queue = ATA_MAX_QUEUE,
430 .this_id = ATA_SHT_THIS_ID,
431 .sg_tablesize = LIBATA_MAX_PRD,
432 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
433 .emulated = ATA_SHT_EMULATED,
434 .use_clustering = ATA_SHT_USE_CLUSTERING,
435 .proc_name = DRV_NAME,
436 .dma_boundary = ATA_DMA_BOUNDARY,
437 .slave_configure = nv_swncq_slave_config,
438 .slave_destroy = ata_scsi_slave_destroy,
439 .bios_param = ata_std_bios_param,
440};
441
ada364e8 442static const struct ata_port_operations nv_generic_ops = {
1da177e4
LT		 443 .tf_load = ata_tf_load,
444 .tf_read = ata_tf_read,
445 .exec_command = ata_exec_command,
446 .check_status = ata_check_status,
447 .dev_select = ata_std_dev_select,
1da177e4
LT		 448 .bmdma_setup = ata_bmdma_setup,
449 .bmdma_start = ata_bmdma_start,
450 .bmdma_stop = ata_bmdma_stop,
451 .bmdma_status = ata_bmdma_status,
452 .qc_prep = ata_qc_prep,
453 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 454 .freeze = ata_bmdma_freeze,
455 .thaw = ata_bmdma_thaw,
456 .error_handler = nv_error_handler,
457 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 458 .data_xfer = ata_data_xfer,
1da177e4 459 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 460 .irq_on = ata_irq_on,
1da177e4
LT		 461 .scr_read = nv_scr_read,
462 .scr_write = nv_scr_write,
463 .port_start = ata_port_start,
1da177e4
LT		 464};
465
ada364e8 466static const struct ata_port_operations nv_nf2_ops = {
ada364e8
TH		 467 .tf_load = ata_tf_load,
468 .tf_read = ata_tf_read,
469 .exec_command = ata_exec_command,
470 .check_status = ata_check_status,
471 .dev_select = ata_std_dev_select,
ada364e8
TH		 472 .bmdma_setup = ata_bmdma_setup,
473 .bmdma_start = ata_bmdma_start,
474 .bmdma_stop = ata_bmdma_stop,
475 .bmdma_status = ata_bmdma_status,
476 .qc_prep = ata_qc_prep,
477 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 478 .freeze = nv_nf2_freeze,
479 .thaw = nv_nf2_thaw,
480 .error_handler = nv_error_handler,
481 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 482 .data_xfer = ata_data_xfer,
ada364e8 483 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 484 .irq_on = ata_irq_on,
ada364e8
TH		 485 .scr_read = nv_scr_read,
486 .scr_write = nv_scr_write,
487 .port_start = ata_port_start,
ada364e8
TH		 488};
489
490static const struct ata_port_operations nv_ck804_ops = {
ada364e8
TH		 491 .tf_load = ata_tf_load,
492 .tf_read = ata_tf_read,
493 .exec_command = ata_exec_command,
494 .check_status = ata_check_status,
495 .dev_select = ata_std_dev_select,
ada364e8
TH		 496 .bmdma_setup = ata_bmdma_setup,
497 .bmdma_start = ata_bmdma_start,
498 .bmdma_stop = ata_bmdma_stop,
499 .bmdma_status = ata_bmdma_status,
500 .qc_prep = ata_qc_prep,
501 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 502 .freeze = nv_ck804_freeze,
503 .thaw = nv_ck804_thaw,
504 .error_handler = nv_error_handler,
505 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 506 .data_xfer = ata_data_xfer,
ada364e8 507 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 508 .irq_on = ata_irq_on,
ada364e8
TH		 509 .scr_read = nv_scr_read,
510 .scr_write = nv_scr_write,
511 .port_start = ata_port_start,
ada364e8
TH		 512 .host_stop = nv_ck804_host_stop,
513};
514
fbbb262d 515static const struct ata_port_operations nv_adma_ops = {
fbbb262d 516 .tf_load = ata_tf_load,
f2fb344b 517 .tf_read = nv_adma_tf_read,
2dec7555 518 .check_atapi_dma = nv_adma_check_atapi_dma,
fbbb262d
RH		 519 .exec_command = ata_exec_command,
520 .check_status = ata_check_status,
521 .dev_select = ata_std_dev_select,
f5ecac2d
RH		 522 .bmdma_setup = ata_bmdma_setup,
523 .bmdma_start = ata_bmdma_start,
524 .bmdma_stop = ata_bmdma_stop,
525 .bmdma_status = ata_bmdma_status,
31cc23b3 526 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 527 .qc_prep = nv_adma_qc_prep,
528 .qc_issue = nv_adma_qc_issue,
53014e25
RH		 529 .freeze = nv_adma_freeze,
530 .thaw = nv_adma_thaw,
fbbb262d 531 .error_handler = nv_adma_error_handler,
f5ecac2d 532 .post_internal_cmd = nv_adma_post_internal_cmd,
0d5ff566 533 .data_xfer = ata_data_xfer,
fbbb262d 534 .irq_clear = nv_adma_irq_clear,
246ce3b6 535 .irq_on = ata_irq_on,
fbbb262d
RH		 536 .scr_read = nv_scr_read,
537 .scr_write = nv_scr_write,
538 .port_start = nv_adma_port_start,
539 .port_stop = nv_adma_port_stop,
438ac6d5 540#ifdef CONFIG_PM
cdf56bcf
RH		 541 .port_suspend = nv_adma_port_suspend,
542 .port_resume = nv_adma_port_resume,
438ac6d5 543#endif
fbbb262d
RH		 544 .host_stop = nv_adma_host_stop,
545};
546
f140f0f1
KL		 547static const struct ata_port_operations nv_swncq_ops = {
548 .tf_load = ata_tf_load,
549 .tf_read = ata_tf_read,
550 .exec_command = ata_exec_command,
551 .check_status = ata_check_status,
552 .dev_select = ata_std_dev_select,
553 .bmdma_setup = ata_bmdma_setup,
554 .bmdma_start = ata_bmdma_start,
555 .bmdma_stop = ata_bmdma_stop,
556 .bmdma_status = ata_bmdma_status,
557 .qc_defer = ata_std_qc_defer,
558 .qc_prep = nv_swncq_qc_prep,
559 .qc_issue = nv_swncq_qc_issue,
560 .freeze = nv_mcp55_freeze,
561 .thaw = nv_mcp55_thaw,
562 .error_handler = nv_swncq_error_handler,
563 .post_internal_cmd = ata_bmdma_post_internal_cmd,
564 .data_xfer = ata_data_xfer,
565 .irq_clear = ata_bmdma_irq_clear,
566 .irq_on = ata_irq_on,
567 .scr_read = nv_scr_read,
568 .scr_write = nv_scr_write,
569#ifdef CONFIG_PM
570 .port_suspend = nv_swncq_port_suspend,
571 .port_resume = nv_swncq_port_resume,
572#endif
573 .port_start = nv_swncq_port_start,
574};
575
1626aeb8 576static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 577 /* generic */
578 {
579 .sht = &nv_sht,
0c88758b
TH		 580 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
581 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 582 .pio_mask = NV_PIO_MASK,
583 .mwdma_mask = NV_MWDMA_MASK,
584 .udma_mask = NV_UDMA_MASK,
585 .port_ops = &nv_generic_ops,
9a829ccf 586 .irq_handler = nv_generic_interrupt,
ada364e8
TH		 587 },
588 /* nforce2/3 */
589 {
590 .sht = &nv_sht,
0c88758b
TH		 591 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
592 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 593 .pio_mask = NV_PIO_MASK,
594 .mwdma_mask = NV_MWDMA_MASK,
595 .udma_mask = NV_UDMA_MASK,
596 .port_ops = &nv_nf2_ops,
9a829ccf 597 .irq_handler = nv_nf2_interrupt,
ada364e8
TH		 598 },
599 /* ck804 */
600 {
601 .sht = &nv_sht,
0c88758b
TH		 602 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
603 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 604 .pio_mask = NV_PIO_MASK,
605 .mwdma_mask = NV_MWDMA_MASK,
606 .udma_mask = NV_UDMA_MASK,
607 .port_ops = &nv_ck804_ops,
9a829ccf 608 .irq_handler = nv_ck804_interrupt,
ada364e8 609 },
fbbb262d
RH		 610 /* ADMA */
611 {
612 .sht = &nv_adma_sht,
613 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
614 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
0c88758b 615 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
fbbb262d
RH		 616 .pio_mask = NV_PIO_MASK,
617 .mwdma_mask = NV_MWDMA_MASK,
618 .udma_mask = NV_UDMA_MASK,
619 .port_ops = &nv_adma_ops,
9a829ccf 620 .irq_handler = nv_adma_interrupt,
fbbb262d 621 },
f140f0f1
KL		 622 /* SWNCQ */
623 {
624 .sht = &nv_swncq_sht,
625 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
626 ATA_FLAG_NCQ,
627 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
628 .pio_mask = NV_PIO_MASK,
629 .mwdma_mask = NV_MWDMA_MASK,
630 .udma_mask = NV_UDMA_MASK,
631 .port_ops = &nv_swncq_ops,
632 .irq_handler = nv_swncq_interrupt,
633 },
1da177e4
LT		 634};
635
636MODULE_AUTHOR("NVIDIA");
637MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
638MODULE_LICENSE("GPL");
639MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
640MODULE_VERSION(DRV_VERSION);
641
fbbb262d 642static int adma_enabled = 1;
f140f0f1 643static int swncq_enabled;
fbbb262d 644
2dec7555
RH		 645static void nv_adma_register_mode(struct ata_port *ap)
646{
2dec7555 647 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 648 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 649 u16 tmp, status;
650 int count = 0;
2dec7555
RH		 651
652 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
653 return;
654
a2cfe81a 655 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 656 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 657 ndelay(50);
658 status = readw(mmio + NV_ADMA_STAT);
659 count++;
660 }
2dcb407e 661 if (count == 20)
a2cfe81a
RH		 662 ata_port_printk(ap, KERN_WARNING,
663 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
664 status);
665
2dec7555
RH		 666 tmp = readw(mmio + NV_ADMA_CTL);
667 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
668
a2cfe81a
RH		 669 count = 0;
670 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 671 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 672 ndelay(50);
673 status = readw(mmio + NV_ADMA_STAT);
674 count++;
675 }
2dcb407e 676 if (count == 20)
a2cfe81a
RH		 677 ata_port_printk(ap, KERN_WARNING,
678 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
679 status);
680
2dec7555
RH		 681 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
682}
683
684static void nv_adma_mode(struct ata_port *ap)
685{
2dec7555 686 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 687 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 688 u16 tmp, status;
689 int count = 0;
2dec7555
RH		 690
691 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
692 return;
f20b16ff 693
2dec7555
RH		 694 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
695
696 tmp = readw(mmio + NV_ADMA_CTL);
697 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
698
a2cfe81a 699 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 700 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 701 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
702 ndelay(50);
703 status = readw(mmio + NV_ADMA_STAT);
704 count++;
705 }
2dcb407e 706 if (count == 20)
a2cfe81a
RH		 707 ata_port_printk(ap, KERN_WARNING,
708 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
709 status);
710
2dec7555
RH		 711 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
712}
713
fbbb262d
RH		 714static int nv_adma_slave_config(struct scsi_device *sdev)
715{
716 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555
RH		 717 struct nv_adma_port_priv *pp = ap->private_data;
718 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
fbbb262d
RH		 719 u64 bounce_limit;
720 unsigned long segment_boundary;
721 unsigned short sg_tablesize;
722 int rc;
2dec7555
RH		 723 int adma_enable;
724 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 725
726 rc = ata_scsi_slave_config(sdev);
727
728 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
729 /* Not a proper libata device, ignore */
730 return rc;
731
9af5c9c9 732 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 733 /*
734 * NVIDIA reports that ADMA mode does not support ATAPI commands.
735 * Therefore ATAPI commands are sent through the legacy interface.
736 * However, the legacy interface only supports 32-bit DMA.
737 * Restrict DMA parameters as required by the legacy interface
738 * when an ATAPI device is connected.
739 */
740 bounce_limit = ATA_DMA_MASK;
741 segment_boundary = ATA_DMA_BOUNDARY;
742 /* Subtract 1 since an extra entry may be needed for padding, see
743 libata-scsi.c */
744 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 745
2dec7555
RH		 746 /* Since the legacy DMA engine is in use, we need to disable ADMA
747 on the port. */
748 adma_enable = 0;
749 nv_adma_register_mode(ap);
2dcb407e 750 } else {
fbbb262d
RH		 751 bounce_limit = *ap->dev->dma_mask;
752 segment_boundary = NV_ADMA_DMA_BOUNDARY;
753 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 754 adma_enable = 1;
fbbb262d 755 }
f20b16ff 756
2dec7555
RH		 757 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
758
2dcb407e 759 if (ap->port_no == 1)
2dec7555
RH		 760 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
761 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
762 else
763 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
764 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 765
2dcb407e 766 if (adma_enable) {
2dec7555
RH		 767 new_reg = current_reg | config_mask;
768 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 769 } else {
2dec7555
RH		 770 new_reg = current_reg & ~config_mask;
771 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
772 }
f20b16ff 773
2dcb407e 774 if (current_reg != new_reg)
2dec7555 775 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 776
fbbb262d
RH		 777 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
778 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
779 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
780 ata_port_printk(ap, KERN_INFO,
781 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
782 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
783 return rc;
784}
785
2dec7555
RH		 786static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
787{
788 struct nv_adma_port_priv *pp = qc->ap->private_data;
789 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
790}
791
f2fb344b
RH		 792static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
793{
3f3debdb
RH		 794 /* Other than when internal or pass-through commands are executed,
795 the only time this function will be called in ADMA mode will be
796 if a command fails. In the failure case we don't care about going
797 into register mode with ADMA commands pending, as the commands will
798 all shortly be aborted anyway. We assume that NCQ commands are not
799 issued via passthrough, which is the only way that switching into
800 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 801 nv_adma_register_mode(ap);
802
803 ata_tf_read(ap, tf);
804}
805
2dec7555 806static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 807{
808 unsigned int idx = 0;
809
2dcb407e 810 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 811 if (tf->flags & ATA_TFLAG_LBA48) {
812 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
813 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
815 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
816 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
818 } else
819 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 820
ac3d6b86
RH		 821 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
823 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
824 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 825 }
a84471fe 826
2dcb407e 827 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 828 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 829
830 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 831
2dcb407e 832 while (idx < 12)
ac3d6b86 833 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 834
835 return idx;
836}
837
5bd28a4b 838static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 839{
840 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 841 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 842
843 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
844
5bd28a4b
RH		 845 if (unlikely((force_err ||
846 flags & (NV_CPB_RESP_ATA_ERR |
847 NV_CPB_RESP_CMD_ERR |
848 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 849 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 850 int freeze = 0;
851
852 ata_ehi_clear_desc(ehi);
2dcb407e 853 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 854 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 855 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 856 ehi->err_mask |= AC_ERR_DEV;
857 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 858 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 859 ehi->err_mask |= AC_ERR_DEV;
860 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 861 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 862 ehi->err_mask |= AC_ERR_SYSTEM;
863 freeze = 1;
864 } else {
865 /* notifier error, but no error in CPB flags? */
b64bbc39 866 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 867 ehi->err_mask |= AC_ERR_OTHER;
868 freeze = 1;
869 }
870 /* Kill all commands. EH will determine what actually failed. */
871 if (freeze)
872 ata_port_freeze(ap);
873 else
874 ata_port_abort(ap);
875 return 1;
fbbb262d 876 }
5bd28a4b 877
f2fb344b 878 if (likely(flags & NV_CPB_RESP_DONE)) {
fbbb262d 879 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
5bd28a4b
RH		 880 VPRINTK("CPB flags done, flags=0x%x\n", flags);
881 if (likely(qc)) {
2dcb407e 882 DPRINTK("Completing qc from tag %d\n", cpb_num);
fbbb262d 883 ata_qc_complete(qc);
2a54cf76 884 } else {
9af5c9c9 885 struct ata_eh_info *ehi = &ap->link.eh_info;
2a54cf76
RH		 886 /* Notifier bits set without a command may indicate the drive
887 is misbehaving. Raise host state machine violation on this
888 condition. */
5796d1c4
JG		 889 ata_port_printk(ap, KERN_ERR,
890 "notifier for tag %d with no cmd?\n",
891 cpb_num);
2a54cf76
RH		 892 ehi->err_mask |= AC_ERR_HSM;
893 ehi->action |= ATA_EH_SOFTRESET;
894 ata_port_freeze(ap);
895 return 1;
fbbb262d
RH		 896 }
897 }
5bd28a4b 898 return 0;
fbbb262d
RH		 899}
900
2dec7555
RH		 901static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
902{
9af5c9c9 903 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 904
905 /* freeze if hotplugged */
906 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
907 ata_port_freeze(ap);
908 return 1;
909 }
910
911 /* bail out if not our interrupt */
912 if (!(irq_stat & NV_INT_DEV))
913 return 0;
914
915 /* DEV interrupt w/ no active qc? */
916 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
917 ata_check_status(ap);
918 return 1;
919 }
920
921 /* handle interrupt */
f740d168 922 return ata_host_intr(ap, qc);
2dec7555
RH		 923}
924
fbbb262d
RH		 925static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
926{
927 struct ata_host *host = dev_instance;
928 int i, handled = 0;
2dec7555 929 u32 notifier_clears[2];
fbbb262d
RH		 930
931 spin_lock(&host->lock);
932
933 for (i = 0; i < host->n_ports; i++) {
934 struct ata_port *ap = host->ports[i];
2dec7555 935 notifier_clears[i] = 0;
fbbb262d
RH		 936
937 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
938 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 939 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 940 u16 status;
941 u32 gen_ctl;
fbbb262d 942 u32 notifier, notifier_error;
a617c09f 943
53014e25
RH		 944 /* if ADMA is disabled, use standard ata interrupt handler */
945 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
946 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
947 >> (NV_INT_PORT_SHIFT * i);
948 handled += nv_host_intr(ap, irq_stat);
949 continue;
950 }
fbbb262d 951
53014e25 952 /* if in ATA register mode, check for standard interrupts */
fbbb262d 953 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
0d5ff566 954 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
2dec7555 955 >> (NV_INT_PORT_SHIFT * i);
2dcb407e 956 if (ata_tag_valid(ap->link.active_tag))
f740d168
RH		 957 /** NV_INT_DEV indication seems unreliable at times
958 at least in ADMA mode. Force it on always when a
959 command is active, to prevent losing interrupts. */
960 irq_stat |= NV_INT_DEV;
2dec7555 961 handled += nv_host_intr(ap, irq_stat);
fbbb262d
RH		 962 }
963
964 notifier = readl(mmio + NV_ADMA_NOTIFIER);
965 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
2dec7555 966 notifier_clears[i] = notifier | notifier_error;
fbbb262d 967
cdf56bcf 968 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
fbbb262d 969
2dcb407e 970 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
fbbb262d
RH		 971 !notifier_error)
972 /* Nothing to do */
973 continue;
974
975 status = readw(mmio + NV_ADMA_STAT);
976
977 /* Clear status. Ensure the controller sees the clearing before we start
978 looking at any of the CPB statuses, so that any CPB completions after
979 this point in the handler will raise another interrupt. */
980 writew(status, mmio + NV_ADMA_STAT);
981 readw(mmio + NV_ADMA_STAT); /* flush posted write */
982 rmb();
983
5bd28a4b
RH		 984 handled++; /* irq handled if we got here */
985
986 /* freeze if hotplugged or controller error */
987 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
988 NV_ADMA_STAT_HOTUNPLUG |
5278b50c
RH		 989 NV_ADMA_STAT_TIMEOUT |
990 NV_ADMA_STAT_SERROR))) {
9af5c9c9 991 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 992
993 ata_ehi_clear_desc(ehi);
2dcb407e 994 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
5bd28a4b
RH		 995 if (status & NV_ADMA_STAT_TIMEOUT) {
996 ehi->err_mask |= AC_ERR_SYSTEM;
b64bbc39 997 ata_ehi_push_desc(ehi, "timeout");
5bd28a4b
RH		 998 } else if (status & NV_ADMA_STAT_HOTPLUG) {
999 ata_ehi_hotplugged(ehi);
b64bbc39 1000 ata_ehi_push_desc(ehi, "hotplug");
5bd28a4b
RH		 1001 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
1002 ata_ehi_hotplugged(ehi);
b64bbc39 1003 ata_ehi_push_desc(ehi, "hot unplug");
5278b50c
RH		 1004 } else if (status & NV_ADMA_STAT_SERROR) {
1005 /* let libata analyze SError and figure out the cause */
b64bbc39
TH		 1006 ata_ehi_push_desc(ehi, "SError");
1007 } else
1008 ata_ehi_push_desc(ehi, "unknown");
fbbb262d 1009 ata_port_freeze(ap);
fbbb262d
RH		 1010 continue;
1011 }
1012
5bd28a4b
RH		 1013 if (status & (NV_ADMA_STAT_DONE |
1014 NV_ADMA_STAT_CPBERR)) {
8ba5e4cb 1015 u32 check_commands;
721449bf 1016 int pos, error = 0;
8ba5e4cb 1017
b447916e 1018 if (ata_tag_valid(ap->link.active_tag))
9af5c9c9 1019 check_commands = 1 << ap->link.active_tag;
8ba5e4cb 1020 else
9af5c9c9 1021 check_commands = ap->link.sactive;
8ba5e4cb 1022
fbbb262d 1023 /** Check CPBs for completed commands */
721449bf
RH		 1024 while ((pos = ffs(check_commands)) && !error) {
1025 pos--;
1026 error = nv_adma_check_cpb(ap, pos,
5796d1c4
JG		 1027 notifier_error & (1 << pos));
1028 check_commands &= ~(1 << pos);
fbbb262d
RH		 1029 }
1030 }
fbbb262d
RH		 1031 }
1032 }
f20b16ff 1033
b447916e 1034 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 1035 /* Note: Both notifier clear registers must be written
1036 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 1037 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1038 writel(notifier_clears[0], pp->notifier_clear_block);
1039 pp = host->ports[1]->private_data;
1040 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 1041 }
fbbb262d
RH		 1042
1043 spin_unlock(&host->lock);
1044
1045 return IRQ_RETVAL(handled);
1046}
1047
53014e25
RH		 1048static void nv_adma_freeze(struct ata_port *ap)
1049{
1050 struct nv_adma_port_priv *pp = ap->private_data;
1051 void __iomem *mmio = pp->ctl_block;
1052 u16 tmp;
1053
1054 nv_ck804_freeze(ap);
1055
1056 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1057 return;
1058
1059 /* clear any outstanding CK804 notifications */
2dcb407e 1060 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1061 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1062
1063 /* Disable interrupt */
1064 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1065 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1066 mmio + NV_ADMA_CTL);
5796d1c4 1067 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1068}
1069
1070static void nv_adma_thaw(struct ata_port *ap)
1071{
1072 struct nv_adma_port_priv *pp = ap->private_data;
1073 void __iomem *mmio = pp->ctl_block;
1074 u16 tmp;
1075
1076 nv_ck804_thaw(ap);
1077
1078 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1079 return;
1080
1081 /* Enable interrupt */
1082 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1083 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1084 mmio + NV_ADMA_CTL);
5796d1c4 1085 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1086}
1087
fbbb262d
RH		 1088static void nv_adma_irq_clear(struct ata_port *ap)
1089{
cdf56bcf
RH		 1090 struct nv_adma_port_priv *pp = ap->private_data;
1091 void __iomem *mmio = pp->ctl_block;
53014e25 1092 u32 notifier_clears[2];
fbbb262d 1093
53014e25
RH		 1094 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1095 ata_bmdma_irq_clear(ap);
1096 return;
1097 }
1098
1099 /* clear any outstanding CK804 notifications */
2dcb407e 1100 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1101 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1102
53014e25
RH		 1103 /* clear ADMA status */
1104 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1105
53014e25
RH		 1106 /* clear notifiers - note both ports need to be written with
1107 something even though we are only clearing on one */
1108 if (ap->port_no == 0) {
1109 notifier_clears[0] = 0xFFFFFFFF;
1110 notifier_clears[1] = 0;
1111 } else {
1112 notifier_clears[0] = 0;
1113 notifier_clears[1] = 0xFFFFFFFF;
1114 }
1115 pp = ap->host->ports[0]->private_data;
1116 writel(notifier_clears[0], pp->notifier_clear_block);
1117 pp = ap->host->ports[1]->private_data;
1118 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1119}
1120
f5ecac2d 1121static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1122{
f5ecac2d 1123 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1124
b447916e 1125 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
f5ecac2d 1126 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1127}
1128
1129static int nv_adma_port_start(struct ata_port *ap)
1130{
1131 struct device *dev = ap->host->dev;
1132 struct nv_adma_port_priv *pp;
1133 int rc;
1134 void *mem;
1135 dma_addr_t mem_dma;
cdf56bcf 1136 void __iomem *mmio;
fbbb262d
RH		 1137 u16 tmp;
1138
1139 VPRINTK("ENTER\n");
1140
1141 rc = ata_port_start(ap);
1142 if (rc)
1143 return rc;
1144
24dc5f33
TH		 1145 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1146 if (!pp)
1147 return -ENOMEM;
fbbb262d 1148
0d5ff566 1149 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1150 ap->port_no * NV_ADMA_PORT_SIZE;
1151 pp->ctl_block = mmio;
0d5ff566 1152 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1153 pp->notifier_clear_block = pp->gen_block +
1154 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1155
24dc5f33
TH		 1156 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1157 &mem_dma, GFP_KERNEL);
1158 if (!mem)
1159 return -ENOMEM;
fbbb262d
RH		 1160 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1161
1162 /*
1163 * First item in chunk of DMA memory:
1164 * 128-byte command parameter block (CPB)
1165 * one for each command tag
1166 */
1167 pp->cpb = mem;
1168 pp->cpb_dma = mem_dma;
1169
1170 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1171 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1172
1173 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1174 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1175
1176 /*
1177 * Second item: block of ADMA_SGTBL_LEN s/g entries
1178 */
1179 pp->aprd = mem;
1180 pp->aprd_dma = mem_dma;
1181
1182 ap->private_data = pp;
1183
1184 /* clear any outstanding interrupt conditions */
1185 writew(0xffff, mmio + NV_ADMA_STAT);
1186
1187 /* initialize port variables */
1188 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1189
1190 /* clear CPB fetch count */
1191 writew(0, mmio + NV_ADMA_CPB_COUNT);
1192
cdf56bcf 1193 /* clear GO for register mode, enable interrupt */
fbbb262d 1194 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1195 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1196 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1197
1198 tmp = readw(mmio + NV_ADMA_CTL);
1199 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1200 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1201 udelay(1);
1202 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1203 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1204
1205 return 0;
fbbb262d
RH		 1206}
1207
1208static void nv_adma_port_stop(struct ata_port *ap)
1209{
fbbb262d 1210 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1211 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1212
1213 VPRINTK("ENTER\n");
fbbb262d 1214 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1215}
1216
438ac6d5 1217#ifdef CONFIG_PM
cdf56bcf
RH		 1218static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1219{
1220 struct nv_adma_port_priv *pp = ap->private_data;
1221 void __iomem *mmio = pp->ctl_block;
1222
1223 /* Go to register mode - clears GO */
1224 nv_adma_register_mode(ap);
1225
1226 /* clear CPB fetch count */
1227 writew(0, mmio + NV_ADMA_CPB_COUNT);
1228
1229 /* disable interrupt, shut down port */
1230 writew(0, mmio + NV_ADMA_CTL);
1231
1232 return 0;
1233}
1234
1235static int nv_adma_port_resume(struct ata_port *ap)
1236{
1237 struct nv_adma_port_priv *pp = ap->private_data;
1238 void __iomem *mmio = pp->ctl_block;
1239 u16 tmp;
1240
1241 /* set CPB block location */
1242 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1243 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1244
1245 /* clear any outstanding interrupt conditions */
1246 writew(0xffff, mmio + NV_ADMA_STAT);
1247
1248 /* initialize port variables */
1249 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1250
1251 /* clear CPB fetch count */
1252 writew(0, mmio + NV_ADMA_CPB_COUNT);
1253
1254 /* clear GO for register mode, enable interrupt */
1255 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1256 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1257 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1258
1259 tmp = readw(mmio + NV_ADMA_CTL);
1260 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1261 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1262 udelay(1);
1263 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1264 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1265
1266 return 0;
1267}
438ac6d5 1268#endif
fbbb262d 1269
9a829ccf 1270static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1271{
9a829ccf
TH		 1272 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1273 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1274
1275 VPRINTK("ENTER\n");
1276
9a829ccf 1277 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1278
0d5ff566
TH		 1279 ioport->cmd_addr = mmio;
1280 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1281 ioport->error_addr =
0d5ff566
TH		 1282 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1283 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1284 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1285 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1286 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1287 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1288 ioport->status_addr =
0d5ff566 1289 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1290 ioport->altstatus_addr =
0d5ff566 1291 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1292}
1293
9a829ccf 1294static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1295{
9a829ccf 1296 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1297 unsigned int i;
1298 u32 tmp32;
1299
1300 VPRINTK("ENTER\n");
1301
1302 /* enable ADMA on the ports */
1303 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1304 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1305 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1306 NV_MCP_SATA_CFG_20_PORT1_EN |
1307 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1308
1309 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1310
9a829ccf
TH		 1311 for (i = 0; i < host->n_ports; i++)
1312 nv_adma_setup_port(host->ports[i]);
fbbb262d 1313
fbbb262d
RH		 1314 return 0;
1315}
1316
1317static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1318 struct scatterlist *sg,
1319 int idx,
1320 struct nv_adma_prd *aprd)
1321{
41949ed5 1322 u8 flags = 0;
fbbb262d
RH		 1323 if (qc->tf.flags & ATA_TFLAG_WRITE)
1324 flags |= NV_APRD_WRITE;
1325 if (idx == qc->n_elem - 1)
1326 flags |= NV_APRD_END;
1327 else if (idx != 4)
1328 flags |= NV_APRD_CONT;
1329
1330 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1331 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1332 aprd->flags = flags;
41949ed5 1333 aprd->packet_len = 0;
fbbb262d
RH		 1334}
1335
1336static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1337{
1338 struct nv_adma_port_priv *pp = qc->ap->private_data;
1339 unsigned int idx;
1340 struct nv_adma_prd *aprd;
1341 struct scatterlist *sg;
1342
1343 VPRINTK("ENTER\n");
1344
1345 idx = 0;
1346
1347 ata_for_each_sg(sg, qc) {
5796d1c4
JG		 1348 aprd = (idx < 5) ? &cpb->aprd[idx] :
1349 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
fbbb262d
RH		 1350 nv_adma_fill_aprd(qc, sg, idx, aprd);
1351 idx++;
1352 }
1353 if (idx > 5)
1354 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
41949ed5
RH		 1355 else
1356 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1357}
1358
382a6652
RH		 1359static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1360{
1361 struct nv_adma_port_priv *pp = qc->ap->private_data;
1362
1363 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1364 or interrupt-driven no-data commands. */
b447916e 1365 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1366 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1367 return 1;
1368
b447916e 1369 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1370 (qc->tf.protocol == ATA_PROT_NODATA))
1371 return 0;
1372
1373 return 1;
1374}
1375
fbbb262d
RH		 1376static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1377{
1378 struct nv_adma_port_priv *pp = qc->ap->private_data;
1379 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1380 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1381 NV_CPB_CTL_IEN;
1382
382a6652 1383 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1384 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1385 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1386 nv_adma_register_mode(qc->ap);
fbbb262d
RH		 1387 ata_qc_prep(qc);
1388 return;
1389 }
1390
41949ed5
RH		 1391 cpb->resp_flags = NV_CPB_RESP_DONE;
1392 wmb();
1393 cpb->ctl_flags = 0;
1394 wmb();
fbbb262d
RH		 1395
1396 cpb->len = 3;
1397 cpb->tag = qc->tag;
1398 cpb->next_cpb_idx = 0;
1399
1400 /* turn on NCQ flags for NCQ commands */
1401 if (qc->tf.protocol == ATA_PROT_NCQ)
1402 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1403
cdf56bcf
RH		 1404 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1405
fbbb262d
RH		 1406 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1407
b447916e 1408 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1409 nv_adma_fill_sg(qc, cpb);
1410 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1411 } else
1412 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1413
5796d1c4
JG		 1414 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1415 until we are finished filling in all of the contents */
fbbb262d
RH		 1416 wmb();
1417 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1418 wmb();
1419 cpb->resp_flags = 0;
fbbb262d
RH		 1420}
1421
1422static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1423{
2dec7555 1424 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1425 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1426 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1427
1428 VPRINTK("ENTER\n");
1429
3f3debdb
RH		 1430 /* We can't handle result taskfile with NCQ commands, since
1431 retrieving the taskfile switches us out of ADMA mode and would abort
1432 existing commands. */
1433 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1434 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1435 ata_dev_printk(qc->dev, KERN_ERR,
1436 "NCQ w/ RESULT_TF not allowed\n");
1437 return AC_ERR_SYSTEM;
1438 }
1439
382a6652 1440 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1441 /* use ATA register mode */
382a6652 1442 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
3f3debdb
RH		 1443 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1444 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d
RH		 1445 nv_adma_register_mode(qc->ap);
1446 return ata_qc_issue_prot(qc);
1447 } else
1448 nv_adma_mode(qc->ap);
1449
1450 /* write append register, command tag in lower 8 bits
1451 and (number of cpbs to append -1) in top 8 bits */
1452 wmb();
5e5c74a5 1453
b447916e 1454 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1455 /* Seems to need some delay before switching between NCQ and
1456 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1457 udelay(20);
1458 pp->last_issue_ncq = curr_ncq;
1459 }
1460
fbbb262d
RH		 1461 writew(qc->tag, mmio + NV_ADMA_APPEND);
1462
5796d1c4 1463 DPRINTK("Issued tag %u\n", qc->tag);
fbbb262d
RH		 1464
1465 return 0;
1466}
1467
7d12e780 1468static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1469{
cca3974e 1470 struct ata_host *host = dev_instance;
1da177e4
LT		 1471 unsigned int i;
1472 unsigned int handled = 0;
1473 unsigned long flags;
1474
cca3974e 1475 spin_lock_irqsave(&host->lock, flags);
1da177e4 1476
cca3974e 1477 for (i = 0; i < host->n_ports; i++) {
1da177e4
LT		 1478 struct ata_port *ap;
1479
cca3974e 1480 ap = host->ports[i];
c1389503 1481 if (ap &&
029f5468 1482 !(ap->flags & ATA_FLAG_DISABLED)) {
1da177e4
LT		 1483 struct ata_queued_cmd *qc;
1484
9af5c9c9 1485 qc = ata_qc_from_tag(ap, ap->link.active_tag);
e50362ec 1486 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1da177e4 1487 handled += ata_host_intr(ap, qc);
b887030a
AC		 1488 else
1489 // No request pending? Clear interrupt status
1490 // anyway, in case there's one pending.
1491 ap->ops->check_status(ap);
1da177e4
LT		 1492 }
1493
1494 }
1495
cca3974e 1496 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1497
1498 return IRQ_RETVAL(handled);
1499}
1500
cca3974e 1501static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1502{
1503 int i, handled = 0;
1504
cca3974e
JG		 1505 for (i = 0; i < host->n_ports; i++) {
1506 struct ata_port *ap = host->ports[i];
ada364e8
TH		 1507
1508 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1509 handled += nv_host_intr(ap, irq_stat);
1510
1511 irq_stat >>= NV_INT_PORT_SHIFT;
1512 }
1513
1514 return IRQ_RETVAL(handled);
1515}
1516
7d12e780 1517static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1518{
cca3974e 1519 struct ata_host *host = dev_instance;
ada364e8
TH		 1520 u8 irq_stat;
1521 irqreturn_t ret;
1522
cca3974e 1523 spin_lock(&host->lock);
0d5ff566 1524 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1525 ret = nv_do_interrupt(host, irq_stat);
1526 spin_unlock(&host->lock);
ada364e8
TH		 1527
1528 return ret;
1529}
1530
7d12e780 1531static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1532{
cca3974e 1533 struct ata_host *host = dev_instance;
ada364e8
TH		 1534 u8 irq_stat;
1535 irqreturn_t ret;
1536
cca3974e 1537 spin_lock(&host->lock);
0d5ff566 1538 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1539 ret = nv_do_interrupt(host, irq_stat);
1540 spin_unlock(&host->lock);
ada364e8
TH		 1541
1542 return ret;
1543}
1544
da3dbb17 1545static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1da177e4 1546{
1da177e4 1547 if (sc_reg > SCR_CONTROL)
da3dbb17 1548 return -EINVAL;
1da177e4 1549
da3dbb17
TH		 1550 *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1551 return 0;
1da177e4
LT		 1552}
1553
da3dbb17 1554static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1da177e4 1555{
1da177e4 1556 if (sc_reg > SCR_CONTROL)
da3dbb17 1557 return -EINVAL;
1da177e4 1558
0d5ff566 1559 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1560 return 0;
1da177e4
LT		 1561}
1562
39f87582
TH		 1563static void nv_nf2_freeze(struct ata_port *ap)
1564{
0d5ff566 1565 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1566 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1567 u8 mask;
1568
0d5ff566 1569 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1570 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1571 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1572}
1573
1574static void nv_nf2_thaw(struct ata_port *ap)
1575{
0d5ff566 1576 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1577 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1578 u8 mask;
1579
0d5ff566 1580 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1581
0d5ff566 1582 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1583 mask |= (NV_INT_MASK << shift);
0d5ff566 1584 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1585}
1586
1587static void nv_ck804_freeze(struct ata_port *ap)
1588{
0d5ff566 1589 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1590 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1591 u8 mask;
1592
1593 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1594 mask &= ~(NV_INT_ALL << shift);
1595 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1596}
1597
1598static void nv_ck804_thaw(struct ata_port *ap)
1599{
0d5ff566 1600 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1601 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1602 u8 mask;
1603
1604 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1605
1606 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1607 mask |= (NV_INT_MASK << shift);
1608 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1609}
1610
f140f0f1
KL		 1611static void nv_mcp55_freeze(struct ata_port *ap)
1612{
1613 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1614 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1615 u32 mask;
1616
1617 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1618
1619 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1620 mask &= ~(NV_INT_ALL_MCP55 << shift);
1621 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1622 ata_bmdma_freeze(ap);
1623}
1624
1625static void nv_mcp55_thaw(struct ata_port *ap)
1626{
1627 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1628 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1629 u32 mask;
1630
1631 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1632
1633 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1634 mask |= (NV_INT_MASK_MCP55 << shift);
1635 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1636 ata_bmdma_thaw(ap);
1637}
1638
cc0680a5 1639static int nv_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1640 unsigned long deadline)
39f87582
TH		 1641{
1642 unsigned int dummy;
1643
1644 /* SATA hardreset fails to retrieve proper device signature on
1645 * some controllers. Don't classify on hardreset. For more
647c595d 1646 * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
39f87582 1647 */
cc0680a5 1648 return sata_std_hardreset(link, &dummy, deadline);
39f87582
TH		 1649}
1650
1651static void nv_error_handler(struct ata_port *ap)
1652{
1653 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1654 nv_hardreset, ata_std_postreset);
1655}
1656
fbbb262d
RH		 1657static void nv_adma_error_handler(struct ata_port *ap)
1658{
1659 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1660 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1661 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1662 int i;
1663 u16 tmp;
a84471fe 1664
b447916e 1665 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1666 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1667 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1668 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1669 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1670 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1671 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1672
5796d1c4
JG		 1673 ata_port_printk(ap, KERN_ERR,
1674 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1675 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1676 "next cpb count 0x%X next cpb idx 0x%x\n",
1677 notifier, notifier_error, gen_ctl, status,
1678 cpb_count, next_cpb_idx);
2cb27853 1679
b447916e 1680 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1681 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1682 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1683 ap->link.sactive & (1 << i))
2cb27853
RH		 1684 ata_port_printk(ap, KERN_ERR,
1685 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1686 i, cpb->ctl_flags, cpb->resp_flags);
1687 }
1688 }
fbbb262d 1689
fbbb262d
RH		 1690 /* Push us back into port register mode for error handling. */
1691 nv_adma_register_mode(ap);
1692
5796d1c4
JG		 1693 /* Mark all of the CPBs as invalid to prevent them from
1694 being executed */
b447916e 1695 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1696 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1697
1698 /* clear CPB fetch count */
1699 writew(0, mmio + NV_ADMA_CPB_COUNT);
1700
1701 /* Reset channel */
1702 tmp = readw(mmio + NV_ADMA_CTL);
1703 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1704 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1705 udelay(1);
1706 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1707 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1708 }
1709
1710 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1711 nv_hardreset, ata_std_postreset);
1712}
1713
f140f0f1
KL		 1714static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1715{
1716 struct nv_swncq_port_priv *pp = ap->private_data;
1717 struct defer_queue *dq = &pp->defer_queue;
1718
1719 /* queue is full */
1720 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1721 dq->defer_bits |= (1 << qc->tag);
1722 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1723}
1724
1725static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1726{
1727 struct nv_swncq_port_priv *pp = ap->private_data;
1728 struct defer_queue *dq = &pp->defer_queue;
1729 unsigned int tag;
1730
1731 if (dq->head == dq->tail) /* null queue */
1732 return NULL;
1733
1734 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1735 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1736 WARN_ON(!(dq->defer_bits & (1 << tag)));
1737 dq->defer_bits &= ~(1 << tag);
1738
1739 return ata_qc_from_tag(ap, tag);
1740}
1741
1742static void nv_swncq_fis_reinit(struct ata_port *ap)
1743{
1744 struct nv_swncq_port_priv *pp = ap->private_data;
1745
1746 pp->dhfis_bits = 0;
1747 pp->dmafis_bits = 0;
1748 pp->sdbfis_bits = 0;
1749 pp->ncq_flags = 0;
1750}
1751
1752static void nv_swncq_pp_reinit(struct ata_port *ap)
1753{
1754 struct nv_swncq_port_priv *pp = ap->private_data;
1755 struct defer_queue *dq = &pp->defer_queue;
1756
1757 dq->head = 0;
1758 dq->tail = 0;
1759 dq->defer_bits = 0;
1760 pp->qc_active = 0;
1761 pp->last_issue_tag = ATA_TAG_POISON;
1762 nv_swncq_fis_reinit(ap);
1763}
1764
1765static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1766{
1767 struct nv_swncq_port_priv *pp = ap->private_data;
1768
1769 writew(fis, pp->irq_block);
1770}
1771
1772static void __ata_bmdma_stop(struct ata_port *ap)
1773{
1774 struct ata_queued_cmd qc;
1775
1776 qc.ap = ap;
1777 ata_bmdma_stop(&qc);
1778}
1779
1780static void nv_swncq_ncq_stop(struct ata_port *ap)
1781{
1782 struct nv_swncq_port_priv *pp = ap->private_data;
1783 unsigned int i;
1784 u32 sactive;
1785 u32 done_mask;
1786
1787 ata_port_printk(ap, KERN_ERR,
1788 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1789 ap->qc_active, ap->link.sactive);
1790 ata_port_printk(ap, KERN_ERR,
1791 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1792 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1793 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1794 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1795
1796 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1797 ap->ops->check_status(ap),
1798 ioread8(ap->ioaddr.error_addr));
1799
1800 sactive = readl(pp->sactive_block);
1801 done_mask = pp->qc_active ^ sactive;
1802
1803 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1804 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1805 u8 err = 0;
1806 if (pp->qc_active & (1 << i))
1807 err = 0;
1808 else if (done_mask & (1 << i))
1809 err = 1;
1810 else
1811 continue;
1812
1813 ata_port_printk(ap, KERN_ERR,
1814 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1815 (pp->dhfis_bits >> i) & 0x1,
1816 (pp->dmafis_bits >> i) & 0x1,
1817 (pp->sdbfis_bits >> i) & 0x1,
1818 (sactive >> i) & 0x1,
1819 (err ? "error! tag doesn't exit" : " "));
1820 }
1821
1822 nv_swncq_pp_reinit(ap);
1823 ap->ops->irq_clear(ap);
1824 __ata_bmdma_stop(ap);
1825 nv_swncq_irq_clear(ap, 0xffff);
1826}
1827
1828static void nv_swncq_error_handler(struct ata_port *ap)
1829{
1830 struct ata_eh_context *ehc = &ap->link.eh_context;
1831
1832 if (ap->link.sactive) {
1833 nv_swncq_ncq_stop(ap);
1834 ehc->i.action |= ATA_EH_HARDRESET;
1835 }
1836
1837 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1838 nv_hardreset, ata_std_postreset);
1839}
1840
1841#ifdef CONFIG_PM
1842static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1843{
1844 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1845 u32 tmp;
1846
1847 /* clear irq */
1848 writel(~0, mmio + NV_INT_STATUS_MCP55);
1849
1850 /* disable irq */
1851 writel(0, mmio + NV_INT_ENABLE_MCP55);
1852
1853 /* disable swncq */
1854 tmp = readl(mmio + NV_CTL_MCP55);
1855 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1856 writel(tmp, mmio + NV_CTL_MCP55);
1857
1858 return 0;
1859}
1860
1861static int nv_swncq_port_resume(struct ata_port *ap)
1862{
1863 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1864 u32 tmp;
1865
1866 /* clear irq */
1867 writel(~0, mmio + NV_INT_STATUS_MCP55);
1868
1869 /* enable irq */
1870 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1871
1872 /* enable swncq */
1873 tmp = readl(mmio + NV_CTL_MCP55);
1874 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1875
1876 return 0;
1877}
1878#endif
1879
1880static void nv_swncq_host_init(struct ata_host *host)
1881{
1882 u32 tmp;
1883 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1884 struct pci_dev *pdev = to_pci_dev(host->dev);
1885 u8 regval;
1886
1887 /* disable ECO 398 */
1888 pci_read_config_byte(pdev, 0x7f, &regval);
1889 regval &= ~(1 << 7);
1890 pci_write_config_byte(pdev, 0x7f, regval);
1891
1892 /* enable swncq */
1893 tmp = readl(mmio + NV_CTL_MCP55);
1894 VPRINTK("HOST_CTL:0x%X\n", tmp);
1895 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1896
1897 /* enable irq intr */
1898 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1899 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1900 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1901
1902 /* clear port irq */
1903 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1904}
1905
1906static int nv_swncq_slave_config(struct scsi_device *sdev)
1907{
1908 struct ata_port *ap = ata_shost_to_port(sdev->host);
1909 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1910 struct ata_device *dev;
1911 int rc;
1912 u8 rev;
1913 u8 check_maxtor = 0;
1914 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1915
1916 rc = ata_scsi_slave_config(sdev);
1917 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1918 /* Not a proper libata device, ignore */
1919 return rc;
1920
1921 dev = &ap->link.device[sdev->id];
1922 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1923 return rc;
1924
1925 /* if MCP51 and Maxtor, then disable ncq */
1926 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1927 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1928 check_maxtor = 1;
1929
1930 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1931 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1932 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1933 pci_read_config_byte(pdev, 0x8, &rev);
1934 if (rev <= 0xa2)
1935 check_maxtor = 1;
1936 }
1937
1938 if (!check_maxtor)
1939 return rc;
1940
1941 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1942
1943 if (strncmp(model_num, "Maxtor", 6) == 0) {
1944 ata_scsi_change_queue_depth(sdev, 1);
1945 ata_dev_printk(dev, KERN_NOTICE,
1946 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1947 }
1948
1949 return rc;
1950}
1951
1952static int nv_swncq_port_start(struct ata_port *ap)
1953{
1954 struct device *dev = ap->host->dev;
1955 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1956 struct nv_swncq_port_priv *pp;
1957 int rc;
1958
1959 rc = ata_port_start(ap);
1960 if (rc)
1961 return rc;
1962
1963 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1964 if (!pp)
1965 return -ENOMEM;
1966
1967 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1968 &pp->prd_dma, GFP_KERNEL);
1969 if (!pp->prd)
1970 return -ENOMEM;
1971 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1972
1973 ap->private_data = pp;
1974 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1975 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1976 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1977
1978 return 0;
1979}
1980
1981static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1982{
1983 if (qc->tf.protocol != ATA_PROT_NCQ) {
1984 ata_qc_prep(qc);
1985 return;
1986 }
1987
1988 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1989 return;
1990
1991 nv_swncq_fill_sg(qc);
1992}
1993
1994static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1995{
1996 struct ata_port *ap = qc->ap;
1997 struct scatterlist *sg;
1998 unsigned int idx;
1999 struct nv_swncq_port_priv *pp = ap->private_data;
2000 struct ata_prd *prd;
2001
2002 WARN_ON(qc->__sg == NULL);
2003 WARN_ON(qc->n_elem == 0 && qc->pad_len == 0);
2004
2005 prd = pp->prd + ATA_MAX_PRD * qc->tag;
2006
2007 idx = 0;
2008 ata_for_each_sg(sg, qc) {
2009 u32 addr, offset;
2010 u32 sg_len, len;
2011
2012 addr = (u32)sg_dma_address(sg);
2013 sg_len = sg_dma_len(sg);
2014
2015 while (sg_len) {
2016 offset = addr & 0xffff;
2017 len = sg_len;
2018 if ((offset + sg_len) > 0x10000)
2019 len = 0x10000 - offset;
2020
2021 prd[idx].addr = cpu_to_le32(addr);
2022 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2023
2024 idx++;
2025 sg_len -= len;
2026 addr += len;
2027 }
2028 }
2029
2030 if (idx)
2031 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2032}
2033
2034static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2035 struct ata_queued_cmd *qc)
2036{
2037 struct nv_swncq_port_priv *pp = ap->private_data;
2038
2039 if (qc == NULL)
2040 return 0;
2041
2042 DPRINTK("Enter\n");
2043
2044 writel((1 << qc->tag), pp->sactive_block);
2045 pp->last_issue_tag = qc->tag;
2046 pp->dhfis_bits &= ~(1 << qc->tag);
2047 pp->dmafis_bits &= ~(1 << qc->tag);
2048 pp->qc_active |= (0x1 << qc->tag);
2049
2050 ap->ops->tf_load(ap, &qc->tf); /* load tf registers */
2051 ap->ops->exec_command(ap, &qc->tf);
2052
2053 DPRINTK("Issued tag %u\n", qc->tag);
2054
2055 return 0;
2056}
2057
2058static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2059{
2060 struct ata_port *ap = qc->ap;
2061 struct nv_swncq_port_priv *pp = ap->private_data;
2062
2063 if (qc->tf.protocol != ATA_PROT_NCQ)
2064 return ata_qc_issue_prot(qc);
2065
2066 DPRINTK("Enter\n");
2067
2068 if (!pp->qc_active)
2069 nv_swncq_issue_atacmd(ap, qc);
2070 else
2071 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2072
2073 return 0;
2074}
2075
2076static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2077{
2078 u32 serror;
2079 struct ata_eh_info *ehi = &ap->link.eh_info;
2080
2081 ata_ehi_clear_desc(ehi);
2082
2083 /* AHCI needs SError cleared; otherwise, it might lock up */
2084 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2085 sata_scr_write(&ap->link, SCR_ERROR, serror);
2086
2087 /* analyze @irq_stat */
2088 if (fis & NV_SWNCQ_IRQ_ADDED)
2089 ata_ehi_push_desc(ehi, "hot plug");
2090 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2091 ata_ehi_push_desc(ehi, "hot unplug");
2092
2093 ata_ehi_hotplugged(ehi);
2094
2095 /* okay, let's hand over to EH */
2096 ehi->serror |= serror;
2097
2098 ata_port_freeze(ap);
2099}
2100
2101static int nv_swncq_sdbfis(struct ata_port *ap)
2102{
2103 struct ata_queued_cmd *qc;
2104 struct nv_swncq_port_priv *pp = ap->private_data;
2105 struct ata_eh_info *ehi = &ap->link.eh_info;
2106 u32 sactive;
2107 int nr_done = 0;
2108 u32 done_mask;
2109 int i;
2110 u8 host_stat;
2111 u8 lack_dhfis = 0;
2112
2113 host_stat = ap->ops->bmdma_status(ap);
2114 if (unlikely(host_stat & ATA_DMA_ERR)) {
2115 /* error when transfering data to/from memory */
2116 ata_ehi_clear_desc(ehi);
2117 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2118 ehi->err_mask |= AC_ERR_HOST_BUS;
2119 ehi->action |= ATA_EH_SOFTRESET;
2120 return -EINVAL;
2121 }
2122
2123 ap->ops->irq_clear(ap);
2124 __ata_bmdma_stop(ap);
2125
2126 sactive = readl(pp->sactive_block);
2127 done_mask = pp->qc_active ^ sactive;
2128
2129 if (unlikely(done_mask & sactive)) {
2130 ata_ehi_clear_desc(ehi);
2131 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2132 "(%08x->%08x)", pp->qc_active, sactive);
2133 ehi->err_mask |= AC_ERR_HSM;
2134 ehi->action |= ATA_EH_HARDRESET;
2135 return -EINVAL;
2136 }
2137 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2138 if (!(done_mask & (1 << i)))
2139 continue;
2140
2141 qc = ata_qc_from_tag(ap, i);
2142 if (qc) {
2143 ata_qc_complete(qc);
2144 pp->qc_active &= ~(1 << i);
2145 pp->dhfis_bits &= ~(1 << i);
2146 pp->dmafis_bits &= ~(1 << i);
2147 pp->sdbfis_bits |= (1 << i);
2148 nr_done++;
2149 }
2150 }
2151
2152 if (!ap->qc_active) {
2153 DPRINTK("over\n");
2154 nv_swncq_pp_reinit(ap);
2155 return nr_done;
2156 }
2157
2158 if (pp->qc_active & pp->dhfis_bits)
2159 return nr_done;
2160
2161 if ((pp->ncq_flags & ncq_saw_backout) ||
2162 (pp->qc_active ^ pp->dhfis_bits))
2163 /* if the controller cann't get a device to host register FIS,
2164 * The driver needs to reissue the new command.
2165 */
2166 lack_dhfis = 1;
2167
2168 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2169 "SWNCQ:qc_active 0x%X defer_bits %X "
2170 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2171 ap->print_id, ap->qc_active, pp->qc_active,
2172 pp->defer_queue.defer_bits, pp->dhfis_bits,
2173 pp->dmafis_bits, pp->last_issue_tag);
2174
2175 nv_swncq_fis_reinit(ap);
2176
2177 if (lack_dhfis) {
2178 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2179 nv_swncq_issue_atacmd(ap, qc);
2180 return nr_done;
2181 }
2182
2183 if (pp->defer_queue.defer_bits) {
2184 /* send deferral queue command */
2185 qc = nv_swncq_qc_from_dq(ap);
2186 WARN_ON(qc == NULL);
2187 nv_swncq_issue_atacmd(ap, qc);
2188 }
2189
2190 return nr_done;
2191}
2192
2193static inline u32 nv_swncq_tag(struct ata_port *ap)
2194{
2195 struct nv_swncq_port_priv *pp = ap->private_data;
2196 u32 tag;
2197
2198 tag = readb(pp->tag_block) >> 2;
2199 return (tag & 0x1f);
2200}
2201
2202static int nv_swncq_dmafis(struct ata_port *ap)
2203{
2204 struct ata_queued_cmd *qc;
2205 unsigned int rw;
2206 u8 dmactl;
2207 u32 tag;
2208 struct nv_swncq_port_priv *pp = ap->private_data;
2209
2210 __ata_bmdma_stop(ap);
2211 tag = nv_swncq_tag(ap);
2212
2213 DPRINTK("dma setup tag 0x%x\n", tag);
2214 qc = ata_qc_from_tag(ap, tag);
2215
2216 if (unlikely(!qc))
2217 return 0;
2218
2219 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2220
2221 /* load PRD table addr. */
2222 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2223 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2224
2225 /* specify data direction, triple-check start bit is clear */
2226 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2227 dmactl &= ~ATA_DMA_WR;
2228 if (!rw)
2229 dmactl |= ATA_DMA_WR;
2230
2231 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2232
2233 return 1;
2234}
2235
2236static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2237{
2238 struct nv_swncq_port_priv *pp = ap->private_data;
2239 struct ata_queued_cmd *qc;
2240 struct ata_eh_info *ehi = &ap->link.eh_info;
2241 u32 serror;
2242 u8 ata_stat;
2243 int rc = 0;
2244
2245 ata_stat = ap->ops->check_status(ap);
2246 nv_swncq_irq_clear(ap, fis);
2247 if (!fis)
2248 return;
2249
2250 if (ap->pflags & ATA_PFLAG_FROZEN)
2251 return;
2252
2253 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2254 nv_swncq_hotplug(ap, fis);
2255 return;
2256 }
2257
2258 if (!pp->qc_active)
2259 return;
2260
2261 if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2262 return;
2263 ap->ops->scr_write(ap, SCR_ERROR, serror);
2264
2265 if (ata_stat & ATA_ERR) {
2266 ata_ehi_clear_desc(ehi);
2267 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2268 ehi->err_mask |= AC_ERR_DEV;
2269 ehi->serror |= serror;
2270 ehi->action |= ATA_EH_SOFTRESET;
2271 ata_port_freeze(ap);
2272 return;
2273 }
2274
2275 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2276 /* If the IRQ is backout, driver must issue
2277 * the new command again some time later.
2278 */
2279 pp->ncq_flags |= ncq_saw_backout;
2280 }
2281
2282 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2283 pp->ncq_flags |= ncq_saw_sdb;
2284 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2285 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2286 ap->print_id, pp->qc_active, pp->dhfis_bits,
2287 pp->dmafis_bits, readl(pp->sactive_block));
2288 rc = nv_swncq_sdbfis(ap);
2289 if (rc < 0)
2290 goto irq_error;
2291 }
2292
2293 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2294 /* The interrupt indicates the new command
2295 * was transmitted correctly to the drive.
2296 */
2297 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2298 pp->ncq_flags |= ncq_saw_d2h;
2299 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2300 ata_ehi_push_desc(ehi, "illegal fis transaction");
2301 ehi->err_mask |= AC_ERR_HSM;
2302 ehi->action |= ATA_EH_HARDRESET;
2303 goto irq_error;
2304 }
2305
2306 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2307 !(pp->ncq_flags & ncq_saw_dmas)) {
2308 ata_stat = ap->ops->check_status(ap);
2309 if (ata_stat & ATA_BUSY)
2310 goto irq_exit;
2311
2312 if (pp->defer_queue.defer_bits) {
2313 DPRINTK("send next command\n");
2314 qc = nv_swncq_qc_from_dq(ap);
2315 nv_swncq_issue_atacmd(ap, qc);
2316 }
2317 }
2318 }
2319
2320 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2321 /* program the dma controller with appropriate PRD buffers
2322 * and start the DMA transfer for requested command.
2323 */
2324 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2325 pp->ncq_flags |= ncq_saw_dmas;
2326 rc = nv_swncq_dmafis(ap);
2327 }
2328
2329irq_exit:
2330 return;
2331irq_error:
2332 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2333 ata_port_freeze(ap);
2334 return;
2335}
2336
2337static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2338{
2339 struct ata_host *host = dev_instance;
2340 unsigned int i;
2341 unsigned int handled = 0;
2342 unsigned long flags;
2343 u32 irq_stat;
2344
2345 spin_lock_irqsave(&host->lock, flags);
2346
2347 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2348
2349 for (i = 0; i < host->n_ports; i++) {
2350 struct ata_port *ap = host->ports[i];
2351
2352 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2353 if (ap->link.sactive) {
2354 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2355 handled = 1;
2356 } else {
2357 if (irq_stat) /* reserve Hotplug */
2358 nv_swncq_irq_clear(ap, 0xfff0);
2359
2360 handled += nv_host_intr(ap, (u8)irq_stat);
2361 }
2362 }
2363 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2364 }
2365
2366 spin_unlock_irqrestore(&host->lock, flags);
2367
2368 return IRQ_RETVAL(handled);
2369}
2370
5796d1c4 2371static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2372{
5796d1c4 2373 static int printed_version;
1626aeb8 2374 const struct ata_port_info *ppi[] = { NULL, NULL };
9a829ccf 2375 struct ata_host *host;
cdf56bcf 2376 struct nv_host_priv *hpriv;
1da177e4
LT		 2377 int rc;
2378 u32 bar;
0d5ff566 2379 void __iomem *base;
fbbb262d 2380 unsigned long type = ent->driver_data;
1da177e4
LT		 2381
2382 // Make sure this is a SATA controller by counting the number of bars
2383 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2384 // it's an IDE controller and we ignore it.
5796d1c4 2385 for (bar = 0; bar < 6; bar++)
1da177e4
LT		 2386 if (pci_resource_start(pdev, bar) == 0)
2387 return -ENODEV;
2388
cdf56bcf 2389 if (!printed_version++)
a9524a76 2390 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 2391
24dc5f33 2392 rc = pcim_enable_device(pdev);
1da177e4 2393 if (rc)
24dc5f33 2394 return rc;
1da177e4 2395
9a829ccf 2396 /* determine type and allocate host */
f140f0f1 2397 if (type == CK804 && adma_enabled) {
fbbb262d
RH		 2398 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2399 type = ADMA;
fbbb262d
RH		 2400 }
2401
360737a9
JG		 2402 if (type == SWNCQ) {
2403 if (swncq_enabled)
2404 dev_printk(KERN_NOTICE, &pdev->dev,
2405 "Using SWNCQ mode\n");
2406 else
2407 type = GENERIC;
2408 }
2409
1626aeb8 2410 ppi[0] = &nv_port_info[type];
d583bc18 2411 rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
9a829ccf
TH		 2412 if (rc)
2413 return rc;
1da177e4 2414
24dc5f33 2415 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2416 if (!hpriv)
24dc5f33 2417 return -ENOMEM;
9a829ccf
TH		 2418 hpriv->type = type;
2419 host->private_data = hpriv;
cdf56bcf 2420
9a829ccf
TH		 2421 /* set 64bit dma masks, may fail */
2422 if (type == ADMA) {
2423 if (pci_set_dma_mask(pdev, DMA_64BIT_MASK) == 0)
2424 pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2425 }
1da177e4 2426
9a829ccf
TH		 2427 /* request and iomap NV_MMIO_BAR */
2428 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2429 if (rc)
2430 return rc;
1da177e4 2431
9a829ccf
TH		 2432 /* configure SCR access */
2433 base = host->iomap[NV_MMIO_BAR];
2434 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2435 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2436
ada364e8 2437 /* enable SATA space for CK804 */
fbbb262d 2438 if (type >= CK804) {
ada364e8
TH		 2439 u8 regval;
2440
2441 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2442 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2443 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2444 }
2445
9a829ccf 2446 /* init ADMA */
fbbb262d 2447 if (type == ADMA) {
9a829ccf 2448 rc = nv_adma_host_init(host);
fbbb262d 2449 if (rc)
24dc5f33 2450 return rc;
360737a9 2451 } else if (type == SWNCQ)
f140f0f1 2452 nv_swncq_host_init(host);
fbbb262d 2453
9a829ccf
TH		 2454 pci_set_master(pdev);
2455 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2456 IRQF_SHARED, ppi[0]->sht);
1da177e4
LT		 2457}
2458
438ac6d5 2459#ifdef CONFIG_PM
cdf56bcf
RH		 2460static int nv_pci_device_resume(struct pci_dev *pdev)
2461{
2462 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2463 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2464 int rc;
cdf56bcf 2465
ce053fa8 2466 rc = ata_pci_device_do_resume(pdev);
b447916e 2467 if (rc)
ce053fa8 2468 return rc;
cdf56bcf
RH		 2469
2470 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2471 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2472 u8 regval;
2473
2474 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2475 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2476 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2477 }
b447916e 2478 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2479 u32 tmp32;
2480 struct nv_adma_port_priv *pp;
2481 /* enable/disable ADMA on the ports appropriately */
2482 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2483
2484 pp = host->ports[0]->private_data;
b447916e 2485 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2486 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2487 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2488 else
2489 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2490 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2491 pp = host->ports[1]->private_data;
b447916e 2492 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2493 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2494 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2495 else
2496 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2497 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2498
2499 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2500 }
2501 }
2502
2503 ata_host_resume(host);
2504
2505 return 0;
2506}
438ac6d5 2507#endif
cdf56bcf 2508
cca3974e 2509static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2510{
cca3974e 2511 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2512 u8 regval;
2513
2514 /* disable SATA space for CK804 */
2515 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2516 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2517 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2518}
2519
fbbb262d
RH		 2520static void nv_adma_host_stop(struct ata_host *host)
2521{
2522 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2523 u32 tmp32;
2524
fbbb262d
RH		 2525 /* disable ADMA on the ports */
2526 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2527 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2528 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2529 NV_MCP_SATA_CFG_20_PORT1_EN |
2530 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2531
2532 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2533
2534 nv_ck804_host_stop(host);
2535}
2536
1da177e4
LT		 2537static int __init nv_init(void)
2538{
b7887196 2539 return pci_register_driver(&nv_pci_driver);
1da177e4
LT		 2540}
2541
2542static void __exit nv_exit(void)
2543{
2544 pci_unregister_driver(&nv_pci_driver);
2545}
2546
2547module_init(nv_init);
2548module_exit(nv_exit);
fbbb262d
RH		 2549module_param_named(adma, adma_enabled, bool, 0444);
2550MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
f140f0f1
KL		 2551module_param_named(swncq, swncq_enabled, bool, 0444);
2552MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2553
Ubuntu Artful kernel mirror