]>
Commit | Line | Data |
---|---|---|
cd248341 JG |
1 | /* |
2 | * Copyright IBM Corp. 2012 | |
3 | * | |
4 | * Author(s): | |
5 | * Jan Glauber <jang@linux.vnet.ibm.com> | |
6 | * | |
7 | * The System z PCI code is a rewrite from a prototype by | |
8 | * the following people (Kudoz!): | |
bedef755 JG |
9 | * Alexander Schmidt |
10 | * Christoph Raisch | |
11 | * Hannes Hering | |
12 | * Hoang-Nam Nguyen | |
13 | * Jan-Bernd Themann | |
14 | * Stefan Roscher | |
15 | * Thomas Klein | |
cd248341 JG |
16 | */ |
17 | ||
896cb7e6 GS |
18 | #define KMSG_COMPONENT "zpci" |
19 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt | |
cd248341 JG |
20 | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/slab.h> | |
23 | #include <linux/err.h> | |
24 | #include <linux/export.h> | |
25 | #include <linux/delay.h> | |
9a4da8a5 JG |
26 | #include <linux/irq.h> |
27 | #include <linux/kernel_stat.h> | |
cd248341 JG |
28 | #include <linux/seq_file.h> |
29 | #include <linux/pci.h> | |
30 | #include <linux/msi.h> | |
31 | ||
9a4da8a5 JG |
32 | #include <asm/isc.h> |
33 | #include <asm/airq.h> | |
cd248341 JG |
34 | #include <asm/facility.h> |
35 | #include <asm/pci_insn.h> | |
a755a45d | 36 | #include <asm/pci_clp.h> |
828b35f6 | 37 | #include <asm/pci_dma.h> |
cd248341 JG |
38 | |
39 | #define DEBUG /* enable pr_debug */ | |
40 | ||
9a4da8a5 JG |
41 | #define SIC_IRQ_MODE_ALL 0 |
42 | #define SIC_IRQ_MODE_SINGLE 1 | |
43 | ||
cd248341 JG |
44 | #define ZPCI_NR_DMA_SPACES 1 |
45 | #define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS | |
46 | ||
47 | /* list of all detected zpci devices */ | |
67f43f38 | 48 | static LIST_HEAD(zpci_list); |
57b5918c | 49 | static DEFINE_SPINLOCK(zpci_list_lock); |
cd248341 | 50 | |
1f44a225 MS |
51 | static struct irq_chip zpci_irq_chip = { |
52 | .name = "zPCI", | |
280510f1 TG |
53 | .irq_unmask = pci_msi_unmask_irq, |
54 | .irq_mask = pci_msi_mask_irq, | |
9a4da8a5 JG |
55 | }; |
56 | ||
1f44a225 MS |
57 | static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); |
58 | static DEFINE_SPINLOCK(zpci_domain_lock); | |
9a4da8a5 | 59 | |
5d0d8f43 | 60 | static struct airq_iv *zpci_aisb_iv; |
1f44a225 | 61 | static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES]; |
9a4da8a5 | 62 | |
f4eae94f MS |
63 | /* Adapter interrupt definitions */ |
64 | static void zpci_irq_handler(struct airq_struct *airq); | |
65 | ||
66 | static struct airq_struct zpci_airq = { | |
67 | .handler = zpci_irq_handler, | |
68 | .isc = PCI_ISC, | |
69 | }; | |
9a4da8a5 | 70 | |
c506fff3 SO |
71 | #define ZPCI_IOMAP_ENTRIES \ |
72 | min(((unsigned long) CONFIG_PCI_NR_FUNCTIONS * PCI_BAR_COUNT), \ | |
73 | ZPCI_IOMAP_MAX_ENTRIES) | |
74 | ||
cd248341 | 75 | static DEFINE_SPINLOCK(zpci_iomap_lock); |
c506fff3 | 76 | static unsigned long *zpci_iomap_bitmap; |
cd248341 JG |
77 | struct zpci_iomap_entry *zpci_iomap_start; |
78 | EXPORT_SYMBOL_GPL(zpci_iomap_start); | |
79 | ||
d0b08853 JG |
80 | static struct kmem_cache *zdev_fmb_cache; |
81 | ||
cd248341 JG |
82 | struct zpci_dev *get_zdev_by_fid(u32 fid) |
83 | { | |
84 | struct zpci_dev *tmp, *zdev = NULL; | |
85 | ||
57b5918c | 86 | spin_lock(&zpci_list_lock); |
cd248341 JG |
87 | list_for_each_entry(tmp, &zpci_list, entry) { |
88 | if (tmp->fid == fid) { | |
89 | zdev = tmp; | |
90 | break; | |
91 | } | |
92 | } | |
57b5918c | 93 | spin_unlock(&zpci_list_lock); |
cd248341 JG |
94 | return zdev; |
95 | } | |
96 | ||
cd248341 JG |
97 | static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) |
98 | { | |
99 | return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; | |
100 | } | |
101 | ||
102 | int pci_domain_nr(struct pci_bus *bus) | |
103 | { | |
104 | return ((struct zpci_dev *) bus->sysdata)->domain; | |
105 | } | |
106 | EXPORT_SYMBOL_GPL(pci_domain_nr); | |
107 | ||
108 | int pci_proc_domain(struct pci_bus *bus) | |
109 | { | |
110 | return pci_domain_nr(bus); | |
111 | } | |
112 | EXPORT_SYMBOL_GPL(pci_proc_domain); | |
113 | ||
9a4da8a5 | 114 | /* Modify PCI: Register adapter interruptions */ |
5d0d8f43 | 115 | static int zpci_set_airq(struct zpci_dev *zdev) |
9a4da8a5 JG |
116 | { |
117 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); | |
cb4deb69 | 118 | struct zpci_fib fib = {0}; |
9a4da8a5 | 119 | |
cb4deb69 SO |
120 | fib.isc = PCI_ISC; |
121 | fib.sum = 1; /* enable summary notifications */ | |
122 | fib.noi = airq_iv_end(zdev->aibv); | |
123 | fib.aibv = (unsigned long) zdev->aibv->vector; | |
124 | fib.aibvo = 0; /* each zdev has its own interrupt vector */ | |
125 | fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8; | |
126 | fib.aisbo = zdev->aisb & 63; | |
9a4da8a5 | 127 | |
cb4deb69 | 128 | return zpci_mod_fc(req, &fib); |
9a4da8a5 JG |
129 | } |
130 | ||
131 | struct mod_pci_args { | |
132 | u64 base; | |
133 | u64 limit; | |
134 | u64 iota; | |
d0b08853 | 135 | u64 fmb_addr; |
9a4da8a5 JG |
136 | }; |
137 | ||
138 | static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args) | |
139 | { | |
140 | u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn); | |
cb4deb69 SO |
141 | struct zpci_fib fib = {0}; |
142 | ||
143 | fib.pba = args->base; | |
144 | fib.pal = args->limit; | |
145 | fib.iota = args->iota; | |
146 | fib.fmb_addr = args->fmb_addr; | |
147 | ||
148 | return zpci_mod_fc(req, &fib); | |
9a4da8a5 JG |
149 | } |
150 | ||
828b35f6 JG |
151 | /* Modify PCI: Register I/O address translation parameters */ |
152 | int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, | |
153 | u64 base, u64 limit, u64 iota) | |
154 | { | |
d0b08853 | 155 | struct mod_pci_args args = { base, limit, iota, 0 }; |
828b35f6 JG |
156 | |
157 | WARN_ON_ONCE(iota & 0x3fff); | |
158 | args.iota |= ZPCI_IOTA_RTTO_FLAG; | |
159 | return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args); | |
160 | } | |
161 | ||
162 | /* Modify PCI: Unregister I/O address translation parameters */ | |
163 | int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) | |
164 | { | |
d0b08853 | 165 | struct mod_pci_args args = { 0, 0, 0, 0 }; |
828b35f6 JG |
166 | |
167 | return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args); | |
168 | } | |
169 | ||
9a4da8a5 | 170 | /* Modify PCI: Unregister adapter interruptions */ |
5d0d8f43 | 171 | static int zpci_clear_airq(struct zpci_dev *zdev) |
9a4da8a5 | 172 | { |
d0b08853 | 173 | struct mod_pci_args args = { 0, 0, 0, 0 }; |
9a4da8a5 JG |
174 | |
175 | return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args); | |
176 | } | |
177 | ||
d0b08853 JG |
178 | /* Modify PCI: Set PCI function measurement parameters */ |
179 | int zpci_fmb_enable_device(struct zpci_dev *zdev) | |
180 | { | |
181 | struct mod_pci_args args = { 0, 0, 0, 0 }; | |
182 | ||
183 | if (zdev->fmb) | |
184 | return -EINVAL; | |
185 | ||
08b42124 | 186 | zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); |
d0b08853 JG |
187 | if (!zdev->fmb) |
188 | return -ENOMEM; | |
d0b08853 JG |
189 | WARN_ON((u64) zdev->fmb & 0xf); |
190 | ||
6001018a SO |
191 | /* reset software counters */ |
192 | atomic64_set(&zdev->allocated_pages, 0); | |
193 | atomic64_set(&zdev->mapped_pages, 0); | |
194 | atomic64_set(&zdev->unmapped_pages, 0); | |
195 | ||
d0b08853 JG |
196 | args.fmb_addr = virt_to_phys(zdev->fmb); |
197 | return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args); | |
198 | } | |
199 | ||
200 | /* Modify PCI: Disable PCI function measurement */ | |
201 | int zpci_fmb_disable_device(struct zpci_dev *zdev) | |
202 | { | |
203 | struct mod_pci_args args = { 0, 0, 0, 0 }; | |
204 | int rc; | |
205 | ||
206 | if (!zdev->fmb) | |
207 | return -EINVAL; | |
208 | ||
209 | /* Function measurement is disabled if fmb address is zero */ | |
210 | rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args); | |
211 | ||
212 | kmem_cache_free(zdev_fmb_cache, zdev->fmb); | |
213 | zdev->fmb = NULL; | |
214 | return rc; | |
215 | } | |
216 | ||
cd248341 JG |
217 | #define ZPCI_PCIAS_CFGSPC 15 |
218 | ||
219 | static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) | |
220 | { | |
221 | u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); | |
222 | u64 data; | |
223 | int rc; | |
224 | ||
9389339f | 225 | rc = zpci_load(&data, req, offset); |
b170bad4 SO |
226 | if (!rc) { |
227 | data = data << ((8 - len) * 8); | |
228 | data = le64_to_cpu(data); | |
cd248341 | 229 | *val = (u32) data; |
b170bad4 | 230 | } else |
cd248341 JG |
231 | *val = 0xffffffff; |
232 | return rc; | |
233 | } | |
234 | ||
235 | static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) | |
236 | { | |
237 | u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); | |
238 | u64 data = val; | |
239 | int rc; | |
240 | ||
241 | data = cpu_to_le64(data); | |
242 | data = data >> ((8 - len) * 8); | |
9389339f | 243 | rc = zpci_store(data, req, offset); |
cd248341 JG |
244 | return rc; |
245 | } | |
246 | ||
b881bc46 | 247 | void pcibios_fixup_bus(struct pci_bus *bus) |
cd248341 JG |
248 | { |
249 | } | |
250 | ||
251 | resource_size_t pcibios_align_resource(void *data, const struct resource *res, | |
252 | resource_size_t size, | |
253 | resource_size_t align) | |
254 | { | |
255 | return 0; | |
256 | } | |
257 | ||
87bc359b JG |
258 | /* combine single writes by using store-block insn */ |
259 | void __iowrite64_copy(void __iomem *to, const void *from, size_t count) | |
260 | { | |
261 | zpci_memcpy_toio(to, from, count); | |
262 | } | |
263 | ||
cd248341 | 264 | /* Create a virtual mapping cookie for a PCI BAR */ |
8cfc99b5 MT |
265 | void __iomem *pci_iomap_range(struct pci_dev *pdev, |
266 | int bar, | |
267 | unsigned long offset, | |
268 | unsigned long max) | |
cd248341 | 269 | { |
198a5278 | 270 | struct zpci_dev *zdev = to_zpci(pdev); |
cd248341 JG |
271 | int idx; |
272 | ||
c0cabadd | 273 | if (!pci_resource_len(pdev, bar)) |
cd248341 JG |
274 | return NULL; |
275 | ||
276 | idx = zdev->bars[bar].map_idx; | |
277 | spin_lock(&zpci_iomap_lock); | |
8cfc99b5 | 278 | /* Detect overrun */ |
f5e44f82 SO |
279 | WARN_ON(!++zpci_iomap_start[idx].count); |
280 | zpci_iomap_start[idx].fh = zdev->fh; | |
281 | zpci_iomap_start[idx].bar = bar; | |
cd248341 JG |
282 | spin_unlock(&zpci_iomap_lock); |
283 | ||
9e00caae | 284 | return (void __iomem *) ZPCI_ADDR(idx) + offset; |
cd248341 | 285 | } |
d9426083 | 286 | EXPORT_SYMBOL(pci_iomap_range); |
8cfc99b5 MT |
287 | |
288 | void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) | |
289 | { | |
290 | return pci_iomap_range(dev, bar, 0, maxlen); | |
291 | } | |
292 | EXPORT_SYMBOL(pci_iomap); | |
cd248341 JG |
293 | |
294 | void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) | |
295 | { | |
9e00caae | 296 | unsigned int idx = ZPCI_IDX(addr); |
cd248341 | 297 | |
cd248341 | 298 | spin_lock(&zpci_iomap_lock); |
8cfc99b5 | 299 | /* Detect underrun */ |
f5e44f82 | 300 | WARN_ON(!zpci_iomap_start[idx].count); |
8cfc99b5 MT |
301 | if (!--zpci_iomap_start[idx].count) { |
302 | zpci_iomap_start[idx].fh = 0; | |
303 | zpci_iomap_start[idx].bar = 0; | |
304 | } | |
cd248341 JG |
305 | spin_unlock(&zpci_iomap_lock); |
306 | } | |
d9426083 | 307 | EXPORT_SYMBOL(pci_iounmap); |
cd248341 JG |
308 | |
309 | static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, | |
310 | int size, u32 *val) | |
311 | { | |
312 | struct zpci_dev *zdev = get_zdev_by_bus(bus); | |
2c3700bb | 313 | int ret; |
cd248341 JG |
314 | |
315 | if (!zdev || devfn != ZPCI_DEVFN) | |
2c3700bb SO |
316 | ret = -ENODEV; |
317 | else | |
318 | ret = zpci_cfg_load(zdev, where, val, size); | |
319 | ||
320 | return ret; | |
cd248341 JG |
321 | } |
322 | ||
323 | static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, | |
324 | int size, u32 val) | |
325 | { | |
326 | struct zpci_dev *zdev = get_zdev_by_bus(bus); | |
2c3700bb | 327 | int ret; |
cd248341 JG |
328 | |
329 | if (!zdev || devfn != ZPCI_DEVFN) | |
2c3700bb SO |
330 | ret = -ENODEV; |
331 | else | |
332 | ret = zpci_cfg_store(zdev, where, val, size); | |
333 | ||
334 | return ret; | |
cd248341 JG |
335 | } |
336 | ||
337 | static struct pci_ops pci_root_ops = { | |
338 | .read = pci_read, | |
339 | .write = pci_write, | |
340 | }; | |
341 | ||
f4eae94f | 342 | static void zpci_irq_handler(struct airq_struct *airq) |
9a4da8a5 | 343 | { |
5d0d8f43 | 344 | unsigned long si, ai; |
1f44a225 | 345 | struct airq_iv *aibv; |
5d0d8f43 | 346 | int irqs_on = 0; |
9a4da8a5 | 347 | |
420f42ec | 348 | inc_irq_stat(IRQIO_PCI); |
5d0d8f43 MS |
349 | for (si = 0;;) { |
350 | /* Scan adapter summary indicator bit vector */ | |
351 | si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv)); | |
352 | if (si == -1UL) { | |
353 | if (irqs_on++) | |
354 | /* End of second scan with interrupts on. */ | |
355 | break; | |
356 | /* First scan complete, reenable interrupts. */ | |
357 | zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); | |
358 | si = 0; | |
359 | continue; | |
360 | } | |
9a4da8a5 | 361 | |
5d0d8f43 | 362 | /* Scan the adapter interrupt vector for this device. */ |
1f44a225 | 363 | aibv = zpci_aibv[si]; |
5d0d8f43 | 364 | for (ai = 0;;) { |
1f44a225 | 365 | ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); |
5d0d8f43 MS |
366 | if (ai == -1UL) |
367 | break; | |
420f42ec | 368 | inc_irq_stat(IRQIO_MSI); |
1f44a225 MS |
369 | airq_iv_lock(aibv, ai); |
370 | generic_handle_irq(airq_iv_get_data(aibv, ai)); | |
371 | airq_iv_unlock(aibv, ai); | |
9a4da8a5 JG |
372 | } |
373 | } | |
5d0d8f43 | 374 | } |
9a4da8a5 | 375 | |
5d0d8f43 | 376 | int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) |
9a4da8a5 | 377 | { |
198a5278 | 378 | struct zpci_dev *zdev = to_zpci(pdev); |
0a0a9421 | 379 | unsigned int hwirq, msi_vecs; |
5d0d8f43 | 380 | unsigned long aisb; |
9a4da8a5 | 381 | struct msi_desc *msi; |
1f44a225 | 382 | struct msi_msg msg; |
0a0a9421 | 383 | int rc, irq; |
9a4da8a5 | 384 | |
a384c892 AG |
385 | if (type == PCI_CAP_ID_MSI && nvec > 1) |
386 | return 1; | |
b19148f6 | 387 | msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); |
9a4da8a5 | 388 | |
5d0d8f43 MS |
389 | /* Allocate adapter summary indicator bit */ |
390 | rc = -EIO; | |
391 | aisb = airq_iv_alloc_bit(zpci_aisb_iv); | |
392 | if (aisb == -1UL) | |
393 | goto out; | |
9a4da8a5 | 394 | zdev->aisb = aisb; |
9a4da8a5 | 395 | |
5d0d8f43 MS |
396 | /* Create adapter interrupt vector */ |
397 | rc = -ENOMEM; | |
1f44a225 | 398 | zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); |
5d0d8f43 MS |
399 | if (!zdev->aibv) |
400 | goto out_si; | |
9a4da8a5 | 401 | |
5d0d8f43 | 402 | /* Wire up shortcut pointer */ |
1f44a225 | 403 | zpci_aibv[aisb] = zdev->aibv; |
5d0d8f43 | 404 | |
1f44a225 MS |
405 | /* Request MSI interrupts */ |
406 | hwirq = 0; | |
df516f42 | 407 | for_each_pci_msi_entry(msi, pdev) { |
1f44a225 MS |
408 | rc = -EIO; |
409 | irq = irq_alloc_desc(0); /* Alloc irq on node 0 */ | |
0a0a9421 | 410 | if (irq < 0) |
1f44a225 MS |
411 | goto out_msi; |
412 | rc = irq_set_msi_desc(irq, msi); | |
9a4da8a5 | 413 | if (rc) |
5d0d8f43 | 414 | goto out_msi; |
1f44a225 MS |
415 | irq_set_chip_and_handler(irq, &zpci_irq_chip, |
416 | handle_simple_irq); | |
417 | msg.data = hwirq; | |
418 | msg.address_lo = zdev->msi_addr & 0xffffffff; | |
419 | msg.address_hi = zdev->msi_addr >> 32; | |
83a18912 | 420 | pci_write_msi_msg(irq, &msg); |
1f44a225 MS |
421 | airq_iv_set_data(zdev->aibv, hwirq, irq); |
422 | hwirq++; | |
9a4da8a5 JG |
423 | } |
424 | ||
5d0d8f43 MS |
425 | /* Enable adapter interrupts */ |
426 | rc = zpci_set_airq(zdev); | |
427 | if (rc) | |
428 | goto out_msi; | |
429 | ||
430 | return (msi_vecs == nvec) ? 0 : msi_vecs; | |
431 | ||
432 | out_msi: | |
df516f42 | 433 | for_each_pci_msi_entry(msi, pdev) { |
1f44a225 | 434 | if (hwirq-- == 0) |
5d0d8f43 | 435 | break; |
1f44a225 MS |
436 | irq_set_msi_desc(msi->irq, NULL); |
437 | irq_free_desc(msi->irq); | |
438 | msi->msg.address_lo = 0; | |
439 | msi->msg.address_hi = 0; | |
440 | msi->msg.data = 0; | |
441 | msi->irq = 0; | |
9a4da8a5 | 442 | } |
1f44a225 | 443 | zpci_aibv[aisb] = NULL; |
5d0d8f43 MS |
444 | airq_iv_release(zdev->aibv); |
445 | out_si: | |
446 | airq_iv_free_bit(zpci_aisb_iv, aisb); | |
447 | out: | |
5d0d8f43 | 448 | return rc; |
9a4da8a5 JG |
449 | } |
450 | ||
5d0d8f43 | 451 | void arch_teardown_msi_irqs(struct pci_dev *pdev) |
9a4da8a5 | 452 | { |
198a5278 | 453 | struct zpci_dev *zdev = to_zpci(pdev); |
9a4da8a5 | 454 | struct msi_desc *msi; |
5d0d8f43 MS |
455 | int rc; |
456 | ||
5d0d8f43 MS |
457 | /* Disable adapter interrupts */ |
458 | rc = zpci_clear_airq(zdev); | |
1f1dcbd4 | 459 | if (rc) |
9a4da8a5 | 460 | return; |
9a4da8a5 | 461 | |
1f44a225 | 462 | /* Release MSI interrupts */ |
df516f42 | 463 | for_each_pci_msi_entry(msi, pdev) { |
8fb878c5 | 464 | if (msi->msi_attrib.is_msix) |
23ed8d57 | 465 | __pci_msix_desc_mask_irq(msi, 1); |
8fb878c5 | 466 | else |
23ed8d57 | 467 | __pci_msi_desc_mask_irq(msi, 1, 1); |
1f44a225 MS |
468 | irq_set_msi_desc(msi->irq, NULL); |
469 | irq_free_desc(msi->irq); | |
470 | msi->msg.address_lo = 0; | |
471 | msi->msg.address_hi = 0; | |
472 | msi->msg.data = 0; | |
473 | msi->irq = 0; | |
474 | } | |
9a4da8a5 | 475 | |
1f44a225 | 476 | zpci_aibv[zdev->aisb] = NULL; |
5d0d8f43 MS |
477 | airq_iv_release(zdev->aibv); |
478 | airq_iv_free_bit(zpci_aisb_iv, zdev->aisb); | |
9a4da8a5 JG |
479 | } |
480 | ||
1803ba2d | 481 | static void zpci_map_resources(struct pci_dev *pdev) |
cd248341 | 482 | { |
cd248341 JG |
483 | resource_size_t len; |
484 | int i; | |
485 | ||
486 | for (i = 0; i < PCI_BAR_COUNT; i++) { | |
487 | len = pci_resource_len(pdev, i); | |
488 | if (!len) | |
489 | continue; | |
5b9f2081 MS |
490 | pdev->resource[i].start = |
491 | (resource_size_t __force) pci_iomap(pdev, i, 0); | |
cd248341 | 492 | pdev->resource[i].end = pdev->resource[i].start + len - 1; |
cd248341 | 493 | } |
944239c5 SO |
494 | } |
495 | ||
1803ba2d | 496 | static void zpci_unmap_resources(struct pci_dev *pdev) |
944239c5 | 497 | { |
944239c5 SO |
498 | resource_size_t len; |
499 | int i; | |
500 | ||
501 | for (i = 0; i < PCI_BAR_COUNT; i++) { | |
502 | len = pci_resource_len(pdev, i); | |
503 | if (!len) | |
504 | continue; | |
5b9f2081 MS |
505 | pci_iounmap(pdev, (void __iomem __force *) |
506 | pdev->resource[i].start); | |
944239c5 SO |
507 | } |
508 | } | |
cd248341 | 509 | |
9a4da8a5 JG |
510 | static int __init zpci_irq_init(void) |
511 | { | |
5d0d8f43 | 512 | int rc; |
9a4da8a5 | 513 | |
f4eae94f MS |
514 | rc = register_adapter_interrupt(&zpci_airq); |
515 | if (rc) | |
5d0d8f43 | 516 | goto out; |
f4eae94f MS |
517 | /* Set summary to 1 to be called every time for the ISC. */ |
518 | *zpci_airq.lsi_ptr = 1; | |
9a4da8a5 | 519 | |
5d0d8f43 MS |
520 | rc = -ENOMEM; |
521 | zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); | |
522 | if (!zpci_aisb_iv) | |
523 | goto out_airq; | |
9a4da8a5 | 524 | |
9389339f | 525 | zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); |
9a4da8a5 JG |
526 | return 0; |
527 | ||
5d0d8f43 MS |
528 | out_airq: |
529 | unregister_adapter_interrupt(&zpci_airq); | |
530 | out: | |
9a4da8a5 JG |
531 | return rc; |
532 | } | |
533 | ||
534 | static void zpci_irq_exit(void) | |
535 | { | |
5d0d8f43 | 536 | airq_iv_release(zpci_aisb_iv); |
f4eae94f | 537 | unregister_adapter_interrupt(&zpci_airq); |
9a4da8a5 JG |
538 | } |
539 | ||
cd248341 JG |
540 | static int zpci_alloc_iomap(struct zpci_dev *zdev) |
541 | { | |
bf19c94d | 542 | unsigned long entry; |
cd248341 JG |
543 | |
544 | spin_lock(&zpci_iomap_lock); | |
c506fff3 SO |
545 | entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); |
546 | if (entry == ZPCI_IOMAP_ENTRIES) { | |
cd248341 JG |
547 | spin_unlock(&zpci_iomap_lock); |
548 | return -ENOSPC; | |
549 | } | |
c506fff3 | 550 | set_bit(entry, zpci_iomap_bitmap); |
cd248341 JG |
551 | spin_unlock(&zpci_iomap_lock); |
552 | return entry; | |
553 | } | |
554 | ||
555 | static void zpci_free_iomap(struct zpci_dev *zdev, int entry) | |
556 | { | |
557 | spin_lock(&zpci_iomap_lock); | |
558 | memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); | |
c506fff3 | 559 | clear_bit(entry, zpci_iomap_bitmap); |
cd248341 JG |
560 | spin_unlock(&zpci_iomap_lock); |
561 | } | |
562 | ||
7a572a3a SO |
563 | static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, |
564 | unsigned long size, unsigned long flags) | |
565 | { | |
566 | struct resource *r; | |
567 | ||
568 | r = kzalloc(sizeof(*r), GFP_KERNEL); | |
569 | if (!r) | |
570 | return NULL; | |
571 | ||
572 | r->start = start; | |
573 | r->end = r->start + size - 1; | |
574 | r->flags = flags; | |
575 | r->name = zdev->res_name; | |
576 | ||
577 | if (request_resource(&iomem_resource, r)) { | |
578 | kfree(r); | |
579 | return NULL; | |
580 | } | |
581 | return r; | |
582 | } | |
583 | ||
584 | static int zpci_setup_bus_resources(struct zpci_dev *zdev, | |
585 | struct list_head *resources) | |
586 | { | |
587 | unsigned long addr, size, flags; | |
588 | struct resource *res; | |
589 | int i, entry; | |
590 | ||
591 | snprintf(zdev->res_name, sizeof(zdev->res_name), | |
592 | "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR); | |
593 | ||
594 | for (i = 0; i < PCI_BAR_COUNT; i++) { | |
595 | if (!zdev->bars[i].size) | |
596 | continue; | |
597 | entry = zpci_alloc_iomap(zdev); | |
598 | if (entry < 0) | |
599 | return entry; | |
600 | zdev->bars[i].map_idx = entry; | |
601 | ||
602 | /* only MMIO is supported */ | |
603 | flags = IORESOURCE_MEM; | |
604 | if (zdev->bars[i].val & 8) | |
605 | flags |= IORESOURCE_PREFETCH; | |
606 | if (zdev->bars[i].val & 4) | |
607 | flags |= IORESOURCE_MEM_64; | |
608 | ||
9e00caae | 609 | addr = ZPCI_ADDR(entry); |
7a572a3a SO |
610 | size = 1UL << zdev->bars[i].size; |
611 | ||
612 | res = __alloc_res(zdev, addr, size, flags); | |
613 | if (!res) { | |
614 | zpci_free_iomap(zdev, entry); | |
615 | return -ENOMEM; | |
616 | } | |
617 | zdev->bars[i].res = res; | |
618 | pci_add_resource(resources, res); | |
619 | } | |
620 | ||
621 | return 0; | |
622 | } | |
623 | ||
624 | static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) | |
625 | { | |
626 | int i; | |
627 | ||
628 | for (i = 0; i < PCI_BAR_COUNT; i++) { | |
2b1df724 | 629 | if (!zdev->bars[i].size || !zdev->bars[i].res) |
7a572a3a SO |
630 | continue; |
631 | ||
632 | zpci_free_iomap(zdev, zdev->bars[i].map_idx); | |
633 | release_resource(zdev->bars[i].res); | |
634 | kfree(zdev->bars[i].res); | |
635 | } | |
636 | } | |
637 | ||
af0a8a84 SO |
638 | int pcibios_add_device(struct pci_dev *pdev) |
639 | { | |
198a5278 | 640 | struct zpci_dev *zdev = to_zpci(pdev); |
cb809182 SO |
641 | struct resource *res; |
642 | int i; | |
643 | ||
644 | zdev->pdev = pdev; | |
ef4858c6 | 645 | pdev->dev.groups = zpci_attr_groups; |
1803ba2d | 646 | zpci_map_resources(pdev); |
cb809182 SO |
647 | |
648 | for (i = 0; i < PCI_BAR_COUNT; i++) { | |
649 | res = &pdev->resource[i]; | |
650 | if (res->parent || !res->flags) | |
651 | continue; | |
652 | pci_claim_resource(pdev, i); | |
653 | } | |
654 | ||
655 | return 0; | |
656 | } | |
657 | ||
1803ba2d SO |
658 | void pcibios_release_device(struct pci_dev *pdev) |
659 | { | |
660 | zpci_unmap_resources(pdev); | |
661 | } | |
662 | ||
cb809182 SO |
663 | int pcibios_enable_device(struct pci_dev *pdev, int mask) |
664 | { | |
198a5278 | 665 | struct zpci_dev *zdev = to_zpci(pdev); |
af0a8a84 | 666 | |
1c21351b | 667 | zdev->pdev = pdev; |
af0a8a84 SO |
668 | zpci_debug_init_device(zdev); |
669 | zpci_fmb_enable_device(zdev); | |
af0a8a84 | 670 | |
d7533232 | 671 | return pci_enable_resources(pdev, mask); |
af0a8a84 SO |
672 | } |
673 | ||
cb809182 | 674 | void pcibios_disable_device(struct pci_dev *pdev) |
944239c5 | 675 | { |
198a5278 | 676 | struct zpci_dev *zdev = to_zpci(pdev); |
944239c5 | 677 | |
944239c5 SO |
678 | zpci_fmb_disable_device(zdev); |
679 | zpci_debug_exit_device(zdev); | |
680 | zdev->pdev = NULL; | |
681 | } | |
682 | ||
69db3b5e SO |
683 | #ifdef CONFIG_HIBERNATE_CALLBACKS |
684 | static int zpci_restore(struct device *dev) | |
685 | { | |
1803ba2d | 686 | struct pci_dev *pdev = to_pci_dev(dev); |
198a5278 | 687 | struct zpci_dev *zdev = to_zpci(pdev); |
69db3b5e SO |
688 | int ret = 0; |
689 | ||
690 | if (zdev->state != ZPCI_FN_STATE_ONLINE) | |
691 | goto out; | |
692 | ||
693 | ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); | |
694 | if (ret) | |
695 | goto out; | |
696 | ||
1803ba2d | 697 | zpci_map_resources(pdev); |
69eea95c | 698 | zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, |
69db3b5e SO |
699 | (u64) zdev->dma_table); |
700 | ||
701 | out: | |
702 | return ret; | |
703 | } | |
704 | ||
705 | static int zpci_freeze(struct device *dev) | |
706 | { | |
1803ba2d | 707 | struct pci_dev *pdev = to_pci_dev(dev); |
198a5278 | 708 | struct zpci_dev *zdev = to_zpci(pdev); |
69db3b5e SO |
709 | |
710 | if (zdev->state != ZPCI_FN_STATE_ONLINE) | |
711 | return 0; | |
712 | ||
713 | zpci_unregister_ioat(zdev, 0); | |
1803ba2d | 714 | zpci_unmap_resources(pdev); |
69db3b5e SO |
715 | return clp_disable_fh(zdev); |
716 | } | |
717 | ||
718 | struct dev_pm_ops pcibios_pm_ops = { | |
719 | .thaw_noirq = zpci_restore, | |
720 | .freeze_noirq = zpci_freeze, | |
721 | .restore_noirq = zpci_restore, | |
722 | .poweroff_noirq = zpci_freeze, | |
723 | }; | |
724 | #endif /* CONFIG_HIBERNATE_CALLBACKS */ | |
725 | ||
cd248341 JG |
726 | static int zpci_alloc_domain(struct zpci_dev *zdev) |
727 | { | |
728 | spin_lock(&zpci_domain_lock); | |
729 | zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); | |
730 | if (zdev->domain == ZPCI_NR_DEVICES) { | |
731 | spin_unlock(&zpci_domain_lock); | |
732 | return -ENOSPC; | |
733 | } | |
734 | set_bit(zdev->domain, zpci_domain); | |
735 | spin_unlock(&zpci_domain_lock); | |
736 | return 0; | |
737 | } | |
738 | ||
739 | static void zpci_free_domain(struct zpci_dev *zdev) | |
740 | { | |
741 | spin_lock(&zpci_domain_lock); | |
742 | clear_bit(zdev->domain, zpci_domain); | |
743 | spin_unlock(&zpci_domain_lock); | |
744 | } | |
745 | ||
7d594322 SO |
746 | void pcibios_remove_bus(struct pci_bus *bus) |
747 | { | |
748 | struct zpci_dev *zdev = get_zdev_by_bus(bus); | |
749 | ||
750 | zpci_exit_slot(zdev); | |
751 | zpci_cleanup_bus_resources(zdev); | |
752 | zpci_free_domain(zdev); | |
753 | ||
754 | spin_lock(&zpci_list_lock); | |
755 | list_del(&zdev->entry); | |
756 | spin_unlock(&zpci_list_lock); | |
757 | ||
758 | kfree(zdev); | |
759 | } | |
760 | ||
761 | static int zpci_scan_bus(struct zpci_dev *zdev) | |
762 | { | |
763 | LIST_HEAD(resources); | |
764 | int ret; | |
765 | ||
766 | ret = zpci_setup_bus_resources(zdev, &resources); | |
767 | if (ret) | |
2b1df724 | 768 | goto error; |
7d594322 SO |
769 | |
770 | zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops, | |
771 | zdev, &resources); | |
772 | if (!zdev->bus) { | |
2b1df724 SO |
773 | ret = -EIO; |
774 | goto error; | |
7d594322 | 775 | } |
7d594322 | 776 | zdev->bus->max_bus_speed = zdev->max_bus_speed; |
b97ea289 | 777 | pci_bus_add_devices(zdev->bus); |
7d594322 | 778 | return 0; |
2b1df724 SO |
779 | |
780 | error: | |
781 | zpci_cleanup_bus_resources(zdev); | |
782 | pci_free_resource_list(&resources); | |
783 | return ret; | |
7d594322 SO |
784 | } |
785 | ||
a755a45d JG |
786 | int zpci_enable_device(struct zpci_dev *zdev) |
787 | { | |
788 | int rc; | |
789 | ||
790 | rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); | |
791 | if (rc) | |
792 | goto out; | |
828b35f6 JG |
793 | |
794 | rc = zpci_dma_init_device(zdev); | |
795 | if (rc) | |
796 | goto out_dma; | |
0ff70ec8 SO |
797 | |
798 | zdev->state = ZPCI_FN_STATE_ONLINE; | |
a755a45d | 799 | return 0; |
828b35f6 JG |
800 | |
801 | out_dma: | |
802 | clp_disable_fh(zdev); | |
a755a45d JG |
803 | out: |
804 | return rc; | |
805 | } | |
806 | EXPORT_SYMBOL_GPL(zpci_enable_device); | |
807 | ||
cb65a669 SO |
808 | int zpci_disable_device(struct zpci_dev *zdev) |
809 | { | |
810 | zpci_dma_exit_device(zdev); | |
811 | return clp_disable_fh(zdev); | |
812 | } | |
813 | EXPORT_SYMBOL_GPL(zpci_disable_device); | |
814 | ||
cd248341 JG |
815 | int zpci_create_device(struct zpci_dev *zdev) |
816 | { | |
817 | int rc; | |
818 | ||
819 | rc = zpci_alloc_domain(zdev); | |
820 | if (rc) | |
821 | goto out; | |
822 | ||
80ed156a | 823 | mutex_init(&zdev->lock); |
1c21351b SO |
824 | if (zdev->state == ZPCI_FN_STATE_CONFIGURED) { |
825 | rc = zpci_enable_device(zdev); | |
826 | if (rc) | |
827 | goto out_free; | |
1c21351b SO |
828 | } |
829 | rc = zpci_scan_bus(zdev); | |
cd248341 | 830 | if (rc) |
1c21351b | 831 | goto out_disable; |
cd248341 | 832 | |
57b5918c | 833 | spin_lock(&zpci_list_lock); |
cd248341 | 834 | list_add_tail(&zdev->entry, &zpci_list); |
57b5918c | 835 | spin_unlock(&zpci_list_lock); |
cd248341 | 836 | |
67f43f38 SO |
837 | zpci_init_slot(zdev); |
838 | ||
cd248341 JG |
839 | return 0; |
840 | ||
1c21351b SO |
841 | out_disable: |
842 | if (zdev->state == ZPCI_FN_STATE_ONLINE) | |
843 | zpci_disable_device(zdev); | |
844 | out_free: | |
cd248341 JG |
845 | zpci_free_domain(zdev); |
846 | out: | |
847 | return rc; | |
848 | } | |
849 | ||
850 | void zpci_stop_device(struct zpci_dev *zdev) | |
851 | { | |
828b35f6 | 852 | zpci_dma_exit_device(zdev); |
cd248341 JG |
853 | /* |
854 | * Note: SCLP disables fh via set-pci-fn so don't | |
855 | * do that here. | |
856 | */ | |
857 | } | |
858 | EXPORT_SYMBOL_GPL(zpci_stop_device); | |
859 | ||
cd248341 JG |
860 | static inline int barsize(u8 size) |
861 | { | |
862 | return (size) ? (1 << size) >> 10 : 0; | |
863 | } | |
864 | ||
865 | static int zpci_mem_init(void) | |
866 | { | |
d0b08853 JG |
867 | zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), |
868 | 16, 0, NULL); | |
869 | if (!zdev_fmb_cache) | |
c506fff3 | 870 | goto error_fmb; |
d0b08853 | 871 | |
c506fff3 SO |
872 | zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, |
873 | sizeof(*zpci_iomap_start), GFP_KERNEL); | |
cd248341 | 874 | if (!zpci_iomap_start) |
9a4da8a5 | 875 | goto error_iomap; |
cd248341 | 876 | |
c506fff3 SO |
877 | zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), |
878 | sizeof(*zpci_iomap_bitmap), GFP_KERNEL); | |
879 | if (!zpci_iomap_bitmap) | |
880 | goto error_iomap_bitmap; | |
881 | ||
882 | return 0; | |
883 | error_iomap_bitmap: | |
884 | kfree(zpci_iomap_start); | |
9a4da8a5 | 885 | error_iomap: |
d0b08853 | 886 | kmem_cache_destroy(zdev_fmb_cache); |
c506fff3 | 887 | error_fmb: |
cd248341 JG |
888 | return -ENOMEM; |
889 | } | |
890 | ||
891 | static void zpci_mem_exit(void) | |
892 | { | |
c506fff3 | 893 | kfree(zpci_iomap_bitmap); |
cd248341 | 894 | kfree(zpci_iomap_start); |
d0b08853 | 895 | kmem_cache_destroy(zdev_fmb_cache); |
cd248341 JG |
896 | } |
897 | ||
257608fb | 898 | static unsigned int s390_pci_probe = 1; |
aa3b7c29 | 899 | static unsigned int s390_pci_initialized; |
cd248341 JG |
900 | |
901 | char * __init pcibios_setup(char *str) | |
902 | { | |
257608fb SO |
903 | if (!strcmp(str, "off")) { |
904 | s390_pci_probe = 0; | |
cd248341 JG |
905 | return NULL; |
906 | } | |
907 | return str; | |
908 | } | |
909 | ||
aa3b7c29 SO |
910 | bool zpci_is_enabled(void) |
911 | { | |
912 | return s390_pci_initialized; | |
913 | } | |
914 | ||
cd248341 JG |
915 | static int __init pci_base_init(void) |
916 | { | |
917 | int rc; | |
918 | ||
1e5635d1 | 919 | if (!s390_pci_probe) |
cd248341 JG |
920 | return 0; |
921 | ||
86cd741b | 922 | if (!test_facility(69) || !test_facility(71) || !test_facility(72)) |
cd248341 JG |
923 | return 0; |
924 | ||
d0b08853 JG |
925 | rc = zpci_debug_init(); |
926 | if (rc) | |
1f44a225 | 927 | goto out; |
d0b08853 | 928 | |
cd248341 JG |
929 | rc = zpci_mem_init(); |
930 | if (rc) | |
931 | goto out_mem; | |
932 | ||
9a4da8a5 JG |
933 | rc = zpci_irq_init(); |
934 | if (rc) | |
935 | goto out_irq; | |
936 | ||
828b35f6 JG |
937 | rc = zpci_dma_init(); |
938 | if (rc) | |
939 | goto out_dma; | |
940 | ||
1d578966 | 941 | rc = clp_scan_pci_devices(); |
a755a45d JG |
942 | if (rc) |
943 | goto out_find; | |
944 | ||
aa3b7c29 | 945 | s390_pci_initialized = 1; |
cd248341 JG |
946 | return 0; |
947 | ||
a755a45d | 948 | out_find: |
828b35f6 JG |
949 | zpci_dma_exit(); |
950 | out_dma: | |
9a4da8a5 JG |
951 | zpci_irq_exit(); |
952 | out_irq: | |
cd248341 JG |
953 | zpci_mem_exit(); |
954 | out_mem: | |
d0b08853 | 955 | zpci_debug_exit(); |
1f44a225 | 956 | out: |
cd248341 JG |
957 | return rc; |
958 | } | |
67f43f38 | 959 | subsys_initcall_sync(pci_base_init); |
57b5918c SO |
960 | |
961 | void zpci_rescan(void) | |
962 | { | |
aa3b7c29 SO |
963 | if (zpci_is_enabled()) |
964 | clp_rescan_pci_devices_simple(); | |
57b5918c | 965 | } |