]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/powerpc/platforms/powernv/pci.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'perf/urgent' into perf/core, to pick up fixes
[mirror_ubuntu-bionic-kernel.git] / arch / powerpc / platforms / powernv / pci.c
1 /*
2 * Support PCI/PCIe on PowerNV platforms
3 *
4 * Currently supports only P5IOC2
5 *
6 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 */
13
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/delay.h>
17 #include <linux/string.h>
18 #include <linux/init.h>
19 #include <linux/bootmem.h>
20 #include <linux/irq.h>
21 #include <linux/io.h>
22 #include <linux/msi.h>
23 #include <linux/iommu.h>
24
25 #include <asm/sections.h>
26 #include <asm/io.h>
27 #include <asm/prom.h>
28 #include <asm/pci-bridge.h>
29 #include <asm/machdep.h>
30 #include <asm/msi_bitmap.h>
31 #include <asm/ppc-pci.h>
32 #include <asm/opal.h>
33 #include <asm/iommu.h>
34 #include <asm/tce.h>
35 #include <asm/firmware.h>
36 #include <asm/eeh_event.h>
37 #include <asm/eeh.h>
38
39 #include "powernv.h"
40 #include "pci.h"
41
42 /* Delay in usec */
43 #define PCI_RESET_DELAY_US 3000000
44
45 #define cfg_dbg(fmt...) do { } while(0)
46 //#define cfg_dbg(fmt...) printk(fmt)
47
48 #ifdef CONFIG_PCI_MSI
49 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
50 {
51 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
52 struct pnv_phb *phb = hose->private_data;
53 struct pci_dn *pdn = pci_get_pdn(pdev);
54 struct msi_desc *entry;
55 struct msi_msg msg;
56 int hwirq;
57 unsigned int virq;
58 int rc;
59
60 if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
61 return -ENODEV;
62
63 if (pdn && pdn->force_32bit_msi && !phb->msi32_support)
64 return -ENODEV;
65
66 list_for_each_entry(entry, &pdev->msi_list, list) {
67 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
68 pr_warn("%s: Supports only 64-bit MSIs\n",
69 pci_name(pdev));
70 return -ENXIO;
71 }
72 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
73 if (hwirq < 0) {
74 pr_warn("%s: Failed to find a free MSI\n",
75 pci_name(pdev));
76 return -ENOSPC;
77 }
78 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
79 if (virq == NO_IRQ) {
80 pr_warn("%s: Failed to map MSI to linux irq\n",
81 pci_name(pdev));
82 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
83 return -ENOMEM;
84 }
85 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
86 virq, entry->msi_attrib.is_64, &msg);
87 if (rc) {
88 pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
89 irq_dispose_mapping(virq);
90 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
91 return rc;
92 }
93 irq_set_msi_desc(virq, entry);
94 write_msi_msg(virq, &msg);
95 }
96 return 0;
97 }
98
99 static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
100 {
101 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
102 struct pnv_phb *phb = hose->private_data;
103 struct msi_desc *entry;
104
105 if (WARN_ON(!phb))
106 return;
107
108 list_for_each_entry(entry, &pdev->msi_list, list) {
109 if (entry->irq == NO_IRQ)
110 continue;
111 irq_set_msi_desc(entry->irq, NULL);
112 msi_bitmap_free_hwirqs(&phb->msi_bmp,
113 virq_to_hw(entry->irq) - phb->msi_base, 1);
114 irq_dispose_mapping(entry->irq);
115 }
116 }
117 #endif /* CONFIG_PCI_MSI */
118
119 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
120 struct OpalIoPhbErrorCommon *common)
121 {
122 struct OpalIoP7IOCPhbErrorData *data;
123 int i;
124
125 data = (struct OpalIoP7IOCPhbErrorData *)common;
126 pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
127 hose->global_number, be32_to_cpu(common->version));
128
129 if (data->brdgCtl)
130 pr_info("brdgCtl: %08x\n",
131 be32_to_cpu(data->brdgCtl));
132 if (data->portStatusReg || data->rootCmplxStatus ||
133 data->busAgentStatus)
134 pr_info("UtlSts: %08x %08x %08x\n",
135 be32_to_cpu(data->portStatusReg),
136 be32_to_cpu(data->rootCmplxStatus),
137 be32_to_cpu(data->busAgentStatus));
138 if (data->deviceStatus || data->slotStatus ||
139 data->linkStatus || data->devCmdStatus ||
140 data->devSecStatus)
141 pr_info("RootSts: %08x %08x %08x %08x %08x\n",
142 be32_to_cpu(data->deviceStatus),
143 be32_to_cpu(data->slotStatus),
144 be32_to_cpu(data->linkStatus),
145 be32_to_cpu(data->devCmdStatus),
146 be32_to_cpu(data->devSecStatus));
147 if (data->rootErrorStatus || data->uncorrErrorStatus ||
148 data->corrErrorStatus)
149 pr_info("RootErrSts: %08x %08x %08x\n",
150 be32_to_cpu(data->rootErrorStatus),
151 be32_to_cpu(data->uncorrErrorStatus),
152 be32_to_cpu(data->corrErrorStatus));
153 if (data->tlpHdr1 || data->tlpHdr2 ||
154 data->tlpHdr3 || data->tlpHdr4)
155 pr_info("RootErrLog: %08x %08x %08x %08x\n",
156 be32_to_cpu(data->tlpHdr1),
157 be32_to_cpu(data->tlpHdr2),
158 be32_to_cpu(data->tlpHdr3),
159 be32_to_cpu(data->tlpHdr4));
160 if (data->sourceId || data->errorClass ||
161 data->correlator)
162 pr_info("RootErrLog1: %08x %016llx %016llx\n",
163 be32_to_cpu(data->sourceId),
164 be64_to_cpu(data->errorClass),
165 be64_to_cpu(data->correlator));
166 if (data->p7iocPlssr || data->p7iocCsr)
167 pr_info("PhbSts: %016llx %016llx\n",
168 be64_to_cpu(data->p7iocPlssr),
169 be64_to_cpu(data->p7iocCsr));
170 if (data->lemFir)
171 pr_info("Lem: %016llx %016llx %016llx\n",
172 be64_to_cpu(data->lemFir),
173 be64_to_cpu(data->lemErrorMask),
174 be64_to_cpu(data->lemWOF));
175 if (data->phbErrorStatus)
176 pr_info("PhbErr: %016llx %016llx %016llx %016llx\n",
177 be64_to_cpu(data->phbErrorStatus),
178 be64_to_cpu(data->phbFirstErrorStatus),
179 be64_to_cpu(data->phbErrorLog0),
180 be64_to_cpu(data->phbErrorLog1));
181 if (data->mmioErrorStatus)
182 pr_info("OutErr: %016llx %016llx %016llx %016llx\n",
183 be64_to_cpu(data->mmioErrorStatus),
184 be64_to_cpu(data->mmioFirstErrorStatus),
185 be64_to_cpu(data->mmioErrorLog0),
186 be64_to_cpu(data->mmioErrorLog1));
187 if (data->dma0ErrorStatus)
188 pr_info("InAErr: %016llx %016llx %016llx %016llx\n",
189 be64_to_cpu(data->dma0ErrorStatus),
190 be64_to_cpu(data->dma0FirstErrorStatus),
191 be64_to_cpu(data->dma0ErrorLog0),
192 be64_to_cpu(data->dma0ErrorLog1));
193 if (data->dma1ErrorStatus)
194 pr_info("InBErr: %016llx %016llx %016llx %016llx\n",
195 be64_to_cpu(data->dma1ErrorStatus),
196 be64_to_cpu(data->dma1FirstErrorStatus),
197 be64_to_cpu(data->dma1ErrorLog0),
198 be64_to_cpu(data->dma1ErrorLog1));
199
200 for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
201 if ((data->pestA[i] >> 63) == 0 &&
202 (data->pestB[i] >> 63) == 0)
203 continue;
204
205 pr_info("PE[%3d] A/B: %016llx %016llx\n",
206 i, be64_to_cpu(data->pestA[i]),
207 be64_to_cpu(data->pestB[i]));
208 }
209 }
210
211 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
212 struct OpalIoPhbErrorCommon *common)
213 {
214 struct OpalIoPhb3ErrorData *data;
215 int i;
216
217 data = (struct OpalIoPhb3ErrorData*)common;
218 pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
219 hose->global_number, be32_to_cpu(common->version));
220 if (data->brdgCtl)
221 pr_info("brdgCtl: %08x\n",
222 be32_to_cpu(data->brdgCtl));
223 if (data->portStatusReg || data->rootCmplxStatus ||
224 data->busAgentStatus)
225 pr_info("UtlSts: %08x %08x %08x\n",
226 be32_to_cpu(data->portStatusReg),
227 be32_to_cpu(data->rootCmplxStatus),
228 be32_to_cpu(data->busAgentStatus));
229 if (data->deviceStatus || data->slotStatus ||
230 data->linkStatus || data->devCmdStatus ||
231 data->devSecStatus)
232 pr_info("RootSts: %08x %08x %08x %08x %08x\n",
233 be32_to_cpu(data->deviceStatus),
234 be32_to_cpu(data->slotStatus),
235 be32_to_cpu(data->linkStatus),
236 be32_to_cpu(data->devCmdStatus),
237 be32_to_cpu(data->devSecStatus));
238 if (data->rootErrorStatus || data->uncorrErrorStatus ||
239 data->corrErrorStatus)
240 pr_info("RootErrSts: %08x %08x %08x\n",
241 be32_to_cpu(data->rootErrorStatus),
242 be32_to_cpu(data->uncorrErrorStatus),
243 be32_to_cpu(data->corrErrorStatus));
244 if (data->tlpHdr1 || data->tlpHdr2 ||
245 data->tlpHdr3 || data->tlpHdr4)
246 pr_info("RootErrLog: %08x %08x %08x %08x\n",
247 be32_to_cpu(data->tlpHdr1),
248 be32_to_cpu(data->tlpHdr2),
249 be32_to_cpu(data->tlpHdr3),
250 be32_to_cpu(data->tlpHdr4));
251 if (data->sourceId || data->errorClass ||
252 data->correlator)
253 pr_info("RootErrLog1: %08x %016llx %016llx\n",
254 be32_to_cpu(data->sourceId),
255 be64_to_cpu(data->errorClass),
256 be64_to_cpu(data->correlator));
257 if (data->nFir)
258 pr_info("nFir: %016llx %016llx %016llx\n",
259 be64_to_cpu(data->nFir),
260 be64_to_cpu(data->nFirMask),
261 be64_to_cpu(data->nFirWOF));
262 if (data->phbPlssr || data->phbCsr)
263 pr_info("PhbSts: %016llx %016llx\n",
264 be64_to_cpu(data->phbPlssr),
265 be64_to_cpu(data->phbCsr));
266 if (data->lemFir)
267 pr_info("Lem: %016llx %016llx %016llx\n",
268 be64_to_cpu(data->lemFir),
269 be64_to_cpu(data->lemErrorMask),
270 be64_to_cpu(data->lemWOF));
271 if (data->phbErrorStatus)
272 pr_info("PhbErr: %016llx %016llx %016llx %016llx\n",
273 be64_to_cpu(data->phbErrorStatus),
274 be64_to_cpu(data->phbFirstErrorStatus),
275 be64_to_cpu(data->phbErrorLog0),
276 be64_to_cpu(data->phbErrorLog1));
277 if (data->mmioErrorStatus)
278 pr_info("OutErr: %016llx %016llx %016llx %016llx\n",
279 be64_to_cpu(data->mmioErrorStatus),
280 be64_to_cpu(data->mmioFirstErrorStatus),
281 be64_to_cpu(data->mmioErrorLog0),
282 be64_to_cpu(data->mmioErrorLog1));
283 if (data->dma0ErrorStatus)
284 pr_info("InAErr: %016llx %016llx %016llx %016llx\n",
285 be64_to_cpu(data->dma0ErrorStatus),
286 be64_to_cpu(data->dma0FirstErrorStatus),
287 be64_to_cpu(data->dma0ErrorLog0),
288 be64_to_cpu(data->dma0ErrorLog1));
289 if (data->dma1ErrorStatus)
290 pr_info("InBErr: %016llx %016llx %016llx %016llx\n",
291 be64_to_cpu(data->dma1ErrorStatus),
292 be64_to_cpu(data->dma1FirstErrorStatus),
293 be64_to_cpu(data->dma1ErrorLog0),
294 be64_to_cpu(data->dma1ErrorLog1));
295
296 for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
297 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
298 (be64_to_cpu(data->pestB[i]) >> 63) == 0)
299 continue;
300
301 pr_info("PE[%3d] A/B: %016llx %016llx\n",
302 i, be64_to_cpu(data->pestA[i]),
303 be64_to_cpu(data->pestB[i]));
304 }
305 }
306
307 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
308 unsigned char *log_buff)
309 {
310 struct OpalIoPhbErrorCommon *common;
311
312 if (!hose || !log_buff)
313 return;
314
315 common = (struct OpalIoPhbErrorCommon *)log_buff;
316 switch (be32_to_cpu(common->ioType)) {
317 case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
318 pnv_pci_dump_p7ioc_diag_data(hose, common);
319 break;
320 case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
321 pnv_pci_dump_phb3_diag_data(hose, common);
322 break;
323 default:
324 pr_warn("%s: Unrecognized ioType %d\n",
325 __func__, be32_to_cpu(common->ioType));
326 }
327 }
328
329 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
330 {
331 unsigned long flags, rc;
332 int has_diag, ret = 0;
333
334 spin_lock_irqsave(&phb->lock, flags);
335
336 /* Fetch PHB diag-data */
337 rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
338 PNV_PCI_DIAG_BUF_SIZE);
339 has_diag = (rc == OPAL_SUCCESS);
340
341 /* If PHB supports compound PE, to handle it */
342 if (phb->unfreeze_pe) {
343 ret = phb->unfreeze_pe(phb,
344 pe_no,
345 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
346 } else {
347 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
348 pe_no,
349 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
350 if (rc) {
351 pr_warn("%s: Failure %ld clearing frozen "
352 "PHB#%x-PE#%x\n",
353 __func__, rc, phb->hose->global_number,
354 pe_no);
355 ret = -EIO;
356 }
357 }
358
359 /*
360 * For now, let's only display the diag buffer when we fail to clear
361 * the EEH status. We'll do more sensible things later when we have
362 * proper EEH support. We need to make sure we don't pollute ourselves
363 * with the normal errors generated when probing empty slots
364 */
365 if (has_diag && ret)
366 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
367
368 spin_unlock_irqrestore(&phb->lock, flags);
369 }
370
371 static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
372 struct device_node *dn)
373 {
374 u8 fstate;
375 __be16 pcierr;
376 int pe_no;
377 s64 rc;
378
379 /*
380 * Get the PE#. During the PCI probe stage, we might not
381 * setup that yet. So all ER errors should be mapped to
382 * reserved PE.
383 */
384 pe_no = PCI_DN(dn)->pe_number;
385 if (pe_no == IODA_INVALID_PE) {
386 if (phb->type == PNV_PHB_P5IOC2)
387 pe_no = 0;
388 else
389 pe_no = phb->ioda.reserved_pe;
390 }
391
392 /*
393 * Fetch frozen state. If the PHB support compound PE,
394 * we need handle that case.
395 */
396 if (phb->get_pe_state) {
397 fstate = phb->get_pe_state(phb, pe_no);
398 } else {
399 rc = opal_pci_eeh_freeze_status(phb->opal_id,
400 pe_no,
401 &fstate,
402 &pcierr,
403 NULL);
404 if (rc) {
405 pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
406 __func__, rc, phb->hose->global_number, pe_no);
407 return;
408 }
409 }
410
411 cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
412 (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
413 pe_no, fstate);
414
415 /* Clear the frozen state if applicable */
416 if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
417 fstate == OPAL_EEH_STOPPED_DMA_FREEZE ||
418 fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
419 /*
420 * If PHB supports compound PE, freeze it for
421 * consistency.
422 */
423 if (phb->freeze_pe)
424 phb->freeze_pe(phb, pe_no);
425
426 pnv_pci_handle_eeh_config(phb, pe_no);
427 }
428 }
429
430 int pnv_pci_cfg_read(struct device_node *dn,
431 int where, int size, u32 *val)
432 {
433 struct pci_dn *pdn = PCI_DN(dn);
434 struct pnv_phb *phb = pdn->phb->private_data;
435 u32 bdfn = (pdn->busno << 8) | pdn->devfn;
436 s64 rc;
437
438 switch (size) {
439 case 1: {
440 u8 v8;
441 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
442 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
443 break;
444 }
445 case 2: {
446 __be16 v16;
447 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
448 &v16);
449 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
450 break;
451 }
452 case 4: {
453 __be32 v32;
454 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
455 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
456 break;
457 }
458 default:
459 return PCIBIOS_FUNC_NOT_SUPPORTED;
460 }
461
462 cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
463 __func__, pdn->busno, pdn->devfn, where, size, *val);
464 return PCIBIOS_SUCCESSFUL;
465 }
466
467 int pnv_pci_cfg_write(struct device_node *dn,
468 int where, int size, u32 val)
469 {
470 struct pci_dn *pdn = PCI_DN(dn);
471 struct pnv_phb *phb = pdn->phb->private_data;
472 u32 bdfn = (pdn->busno << 8) | pdn->devfn;
473
474 cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
475 pdn->busno, pdn->devfn, where, size, val);
476 switch (size) {
477 case 1:
478 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
479 break;
480 case 2:
481 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
482 break;
483 case 4:
484 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
485 break;
486 default:
487 return PCIBIOS_FUNC_NOT_SUPPORTED;
488 }
489
490 return PCIBIOS_SUCCESSFUL;
491 }
492
493 #if CONFIG_EEH
494 static bool pnv_pci_cfg_check(struct pci_controller *hose,
495 struct device_node *dn)
496 {
497 struct eeh_dev *edev = NULL;
498 struct pnv_phb *phb = hose->private_data;
499
500 /* EEH not enabled ? */
501 if (!(phb->flags & PNV_PHB_FLAG_EEH))
502 return true;
503
504 /* PE reset or device removed ? */
505 edev = of_node_to_eeh_dev(dn);
506 if (edev) {
507 if (edev->pe &&
508 (edev->pe->state & EEH_PE_CFG_BLOCKED))
509 return false;
510
511 if (edev->mode & EEH_DEV_REMOVED)
512 return false;
513 }
514
515 return true;
516 }
517 #else
518 static inline pnv_pci_cfg_check(struct pci_controller *hose,
519 struct device_node *dn)
520 {
521 return true;
522 }
523 #endif /* CONFIG_EEH */
524
525 static int pnv_pci_read_config(struct pci_bus *bus,
526 unsigned int devfn,
527 int where, int size, u32 *val)
528 {
529 struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
530 struct pci_dn *pdn;
531 struct pnv_phb *phb;
532 bool found = false;
533 int ret;
534
535 *val = 0xFFFFFFFF;
536 for (dn = busdn->child; dn; dn = dn->sibling) {
537 pdn = PCI_DN(dn);
538 if (pdn && pdn->devfn == devfn) {
539 phb = pdn->phb->private_data;
540 found = true;
541 break;
542 }
543 }
544
545 if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
546 return PCIBIOS_DEVICE_NOT_FOUND;
547
548 ret = pnv_pci_cfg_read(dn, where, size, val);
549 if (phb->flags & PNV_PHB_FLAG_EEH) {
550 if (*val == EEH_IO_ERROR_VALUE(size) &&
551 eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
552 return PCIBIOS_DEVICE_NOT_FOUND;
553 } else {
554 pnv_pci_config_check_eeh(phb, dn);
555 }
556
557 return ret;
558 }
559
560 static int pnv_pci_write_config(struct pci_bus *bus,
561 unsigned int devfn,
562 int where, int size, u32 val)
563 {
564 struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
565 struct pci_dn *pdn;
566 struct pnv_phb *phb;
567 bool found = false;
568 int ret;
569
570 for (dn = busdn->child; dn; dn = dn->sibling) {
571 pdn = PCI_DN(dn);
572 if (pdn && pdn->devfn == devfn) {
573 phb = pdn->phb->private_data;
574 found = true;
575 break;
576 }
577 }
578
579 if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
580 return PCIBIOS_DEVICE_NOT_FOUND;
581
582 ret = pnv_pci_cfg_write(dn, where, size, val);
583 if (!(phb->flags & PNV_PHB_FLAG_EEH))
584 pnv_pci_config_check_eeh(phb, dn);
585
586 return ret;
587 }
588
589 struct pci_ops pnv_pci_ops = {
590 .read = pnv_pci_read_config,
591 .write = pnv_pci_write_config,
592 };
593
594 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
595 unsigned long uaddr, enum dma_data_direction direction,
596 struct dma_attrs *attrs, bool rm)
597 {
598 u64 proto_tce;
599 __be64 *tcep, *tces;
600 u64 rpn;
601
602 proto_tce = TCE_PCI_READ; // Read allowed
603
604 if (direction != DMA_TO_DEVICE)
605 proto_tce |= TCE_PCI_WRITE;
606
607 tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
608 rpn = __pa(uaddr) >> tbl->it_page_shift;
609
610 while (npages--)
611 *(tcep++) = cpu_to_be64(proto_tce |
612 (rpn++ << tbl->it_page_shift));
613
614 /* Some implementations won't cache invalid TCEs and thus may not
615 * need that flush. We'll probably turn it_type into a bit mask
616 * of flags if that becomes the case
617 */
618 if (tbl->it_type & TCE_PCI_SWINV_CREATE)
619 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
620
621 return 0;
622 }
623
624 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
625 unsigned long uaddr,
626 enum dma_data_direction direction,
627 struct dma_attrs *attrs)
628 {
629 return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
630 false);
631 }
632
633 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
634 bool rm)
635 {
636 __be64 *tcep, *tces;
637
638 tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
639
640 while (npages--)
641 *(tcep++) = cpu_to_be64(0);
642
643 if (tbl->it_type & TCE_PCI_SWINV_FREE)
644 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
645 }
646
647 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
648 {
649 pnv_tce_free(tbl, index, npages, false);
650 }
651
652 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
653 {
654 return ((u64 *)tbl->it_base)[index - tbl->it_offset];
655 }
656
657 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
658 unsigned long uaddr,
659 enum dma_data_direction direction,
660 struct dma_attrs *attrs)
661 {
662 return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
663 }
664
665 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
666 {
667 pnv_tce_free(tbl, index, npages, true);
668 }
669
670 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
671 void *tce_mem, u64 tce_size,
672 u64 dma_offset, unsigned page_shift)
673 {
674 tbl->it_blocksize = 16;
675 tbl->it_base = (unsigned long)tce_mem;
676 tbl->it_page_shift = page_shift;
677 tbl->it_offset = dma_offset >> tbl->it_page_shift;
678 tbl->it_index = 0;
679 tbl->it_size = tce_size >> 3;
680 tbl->it_busno = 0;
681 tbl->it_type = TCE_PCI;
682 }
683
684 static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
685 {
686 struct iommu_table *tbl;
687 const __be64 *basep, *swinvp;
688 const __be32 *sizep;
689
690 basep = of_get_property(hose->dn, "linux,tce-base", NULL);
691 sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
692 if (basep == NULL || sizep == NULL) {
693 pr_err("PCI: %s has missing tce entries !\n",
694 hose->dn->full_name);
695 return NULL;
696 }
697 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
698 if (WARN_ON(!tbl))
699 return NULL;
700 pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
701 be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
702 iommu_init_table(tbl, hose->node);
703 iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
704
705 /* Deal with SW invalidated TCEs when needed (BML way) */
706 swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
707 NULL);
708 if (swinvp) {
709 tbl->it_busno = be64_to_cpu(swinvp[1]);
710 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
711 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
712 }
713 return tbl;
714 }
715
716 static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
717 struct pci_dev *pdev)
718 {
719 struct device_node *np = pci_bus_to_OF_node(hose->bus);
720 struct pci_dn *pdn;
721
722 if (np == NULL)
723 return;
724 pdn = PCI_DN(np);
725 if (!pdn->iommu_table)
726 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
727 if (!pdn->iommu_table)
728 return;
729 set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
730 }
731
732 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
733 {
734 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
735 struct pnv_phb *phb = hose->private_data;
736
737 /* If we have no phb structure, try to setup a fallback based on
738 * the device-tree (RTAS PCI for example)
739 */
740 if (phb && phb->dma_dev_setup)
741 phb->dma_dev_setup(phb, pdev);
742 else
743 pnv_pci_dma_fallback_setup(hose, pdev);
744 }
745
746 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
747 {
748 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
749 struct pnv_phb *phb = hose->private_data;
750
751 if (phb && phb->dma_set_mask)
752 return phb->dma_set_mask(phb, pdev, dma_mask);
753 return __dma_set_mask(&pdev->dev, dma_mask);
754 }
755
756 u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
757 {
758 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
759 struct pnv_phb *phb = hose->private_data;
760
761 if (phb && phb->dma_get_required_mask)
762 return phb->dma_get_required_mask(phb, pdev);
763
764 return __dma_get_required_mask(&pdev->dev);
765 }
766
767 void pnv_pci_shutdown(void)
768 {
769 struct pci_controller *hose;
770
771 list_for_each_entry(hose, &hose_list, list_node) {
772 struct pnv_phb *phb = hose->private_data;
773
774 if (phb && phb->shutdown)
775 phb->shutdown(phb);
776 }
777 }
778
779 /* Fixup wrong class code in p7ioc and p8 root complex */
780 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
781 {
782 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
783 }
784 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
785
786 static int pnv_pci_probe_mode(struct pci_bus *bus)
787 {
788 struct pci_controller *hose = pci_bus_to_host(bus);
789 const __be64 *tstamp;
790 u64 now, target;
791
792
793 /* We hijack this as a way to ensure we have waited long
794 * enough since the reset was lifted on the PCI bus
795 */
796 if (bus != hose->bus)
797 return PCI_PROBE_NORMAL;
798 tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
799 if (!tstamp || !*tstamp)
800 return PCI_PROBE_NORMAL;
801
802 now = mftb() / tb_ticks_per_usec;
803 target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
804 + PCI_RESET_DELAY_US;
805
806 pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
807 hose->global_number, target, now);
808
809 if (now < target)
810 msleep((target - now + 999) / 1000);
811
812 return PCI_PROBE_NORMAL;
813 }
814
815 void __init pnv_pci_init(void)
816 {
817 struct device_node *np;
818
819 pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
820
821 /* OPAL absent, try POPAL first then RTAS detection of PHBs */
822 if (!firmware_has_feature(FW_FEATURE_OPAL)) {
823 #ifdef CONFIG_PPC_POWERNV_RTAS
824 init_pci_config_tokens();
825 find_and_init_phbs();
826 #endif /* CONFIG_PPC_POWERNV_RTAS */
827 }
828 /* OPAL is here, do our normal stuff */
829 else {
830 int found_ioda = 0;
831
832 /* Look for IODA IO-Hubs. We don't support mixing IODA
833 * and p5ioc2 due to the need to change some global
834 * probing flags
835 */
836 for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
837 pnv_pci_init_ioda_hub(np);
838 found_ioda = 1;
839 }
840
841 /* Look for p5ioc2 IO-Hubs */
842 if (!found_ioda)
843 for_each_compatible_node(np, NULL, "ibm,p5ioc2")
844 pnv_pci_init_p5ioc2_hub(np);
845
846 /* Look for ioda2 built-in PHB3's */
847 for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
848 pnv_pci_init_ioda2_phb(np);
849 }
850
851 /* Setup the linkage between OF nodes and PHBs */
852 pci_devs_phb_init();
853
854 /* Configure IOMMU DMA hooks */
855 ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
856 ppc_md.tce_build = pnv_tce_build_vm;
857 ppc_md.tce_free = pnv_tce_free_vm;
858 ppc_md.tce_build_rm = pnv_tce_build_rm;
859 ppc_md.tce_free_rm = pnv_tce_free_rm;
860 ppc_md.tce_get = pnv_tce_get;
861 ppc_md.pci_probe_mode = pnv_pci_probe_mode;
862 set_pci_dma_ops(&dma_iommu_ops);
863
864 /* Configure MSIs */
865 #ifdef CONFIG_PCI_MSI
866 ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
867 ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
868 #endif
869 }
870
871 static int tce_iommu_bus_notifier(struct notifier_block *nb,
872 unsigned long action, void *data)
873 {
874 struct device *dev = data;
875
876 switch (action) {
877 case BUS_NOTIFY_ADD_DEVICE:
878 return iommu_add_device(dev);
879 case BUS_NOTIFY_DEL_DEVICE:
880 if (dev->iommu_group)
881 iommu_del_device(dev);
882 return 0;
883 default:
884 return 0;
885 }
886 }
887
888 static struct notifier_block tce_iommu_bus_nb = {
889 .notifier_call = tce_iommu_bus_notifier,
890 };
891
892 static int __init tce_iommu_bus_notifier_init(void)
893 {
894 bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
895 return 0;
896 }
897 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);
ubuntu-bionic-kernel mirror