1 // SPDX-License-Identifier: GPL-2.0-only
3 * IBM Accelerator Family 'GenWQE'
5 * (C) Copyright IBM Corp. 2013
7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9 * Author: Michael Jung <mijung@gmx.net>
10 * Author: Michael Ruettger <michael@ibmra.de>
14 * Module initialization and PCIe setup. Card health monitoring and
15 * recovery functionality. Character device creation and deletion are
16 * controlled from here.
19 #include <linux/types.h>
20 #include <linux/pci.h>
21 #include <linux/err.h>
22 #include <linux/aer.h>
23 #include <linux/string.h>
24 #include <linux/sched.h>
25 #include <linux/wait.h>
26 #include <linux/delay.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/notifier.h>
30 #include <linux/device.h>
31 #include <linux/log2.h>
33 #include "card_base.h"
34 #include "card_ddcb.h"
36 MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");
37 MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");
38 MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");
39 MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");
41 MODULE_DESCRIPTION("GenWQE Card");
42 MODULE_VERSION(DRV_VERSION
);
43 MODULE_LICENSE("GPL");
45 static char genwqe_driver_name
[] = GENWQE_DEVNAME
;
46 static struct class *class_genwqe
;
47 static struct dentry
*debugfs_genwqe
;
48 static struct genwqe_dev
*genwqe_devices
[GENWQE_CARD_NO_MAX
];
50 /* PCI structure for identifying device by PCI vendor and device ID */
51 static const struct pci_device_id genwqe_device_table
[] = {
52 { .vendor
= PCI_VENDOR_ID_IBM
,
53 .device
= PCI_DEVICE_GENWQE
,
54 .subvendor
= PCI_SUBVENDOR_ID_IBM
,
55 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5
,
56 .class = (PCI_CLASSCODE_GENWQE5
<< 8),
60 /* Initial SR-IOV bring-up image */
61 { .vendor
= PCI_VENDOR_ID_IBM
,
62 .device
= PCI_DEVICE_GENWQE
,
63 .subvendor
= PCI_SUBVENDOR_ID_IBM_SRIOV
,
64 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5_SRIOV
,
65 .class = (PCI_CLASSCODE_GENWQE5_SRIOV
<< 8),
69 { .vendor
= PCI_VENDOR_ID_IBM
, /* VF Vendor ID */
70 .device
= 0x0000, /* VF Device ID */
71 .subvendor
= PCI_SUBVENDOR_ID_IBM_SRIOV
,
72 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5_SRIOV
,
73 .class = (PCI_CLASSCODE_GENWQE5_SRIOV
<< 8),
78 { .vendor
= PCI_VENDOR_ID_IBM
,
79 .device
= PCI_DEVICE_GENWQE
,
80 .subvendor
= PCI_SUBVENDOR_ID_IBM_SRIOV
,
81 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5
,
82 .class = (PCI_CLASSCODE_GENWQE5_SRIOV
<< 8),
86 { .vendor
= PCI_VENDOR_ID_IBM
, /* VF Vendor ID */
87 .device
= 0x0000, /* VF Device ID */
88 .subvendor
= PCI_SUBVENDOR_ID_IBM_SRIOV
,
89 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5
,
90 .class = (PCI_CLASSCODE_GENWQE5_SRIOV
<< 8),
94 /* Even one more ... */
95 { .vendor
= PCI_VENDOR_ID_IBM
,
96 .device
= PCI_DEVICE_GENWQE
,
97 .subvendor
= PCI_SUBVENDOR_ID_IBM
,
98 .subdevice
= PCI_SUBSYSTEM_ID_GENWQE5_NEW
,
99 .class = (PCI_CLASSCODE_GENWQE5
<< 8),
103 { 0, } /* 0 terminated list. */
106 MODULE_DEVICE_TABLE(pci
, genwqe_device_table
);
109 * genwqe_dev_alloc() - Create and prepare a new card descriptor
111 * Return: Pointer to card descriptor, or ERR_PTR(err) on error
113 static struct genwqe_dev
*genwqe_dev_alloc(void)
115 unsigned int i
= 0, j
;
116 struct genwqe_dev
*cd
;
118 for (i
= 0; i
< GENWQE_CARD_NO_MAX
; i
++) {
119 if (genwqe_devices
[i
] == NULL
)
122 if (i
>= GENWQE_CARD_NO_MAX
)
123 return ERR_PTR(-ENODEV
);
125 cd
= kzalloc(sizeof(struct genwqe_dev
), GFP_KERNEL
);
127 return ERR_PTR(-ENOMEM
);
130 cd
->class_genwqe
= class_genwqe
;
131 cd
->debugfs_genwqe
= debugfs_genwqe
;
134 * This comes from kernel config option and can be overritten via
137 cd
->use_platform_recovery
= CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY
;
139 init_waitqueue_head(&cd
->queue_waitq
);
141 spin_lock_init(&cd
->file_lock
);
142 INIT_LIST_HEAD(&cd
->file_list
);
144 cd
->card_state
= GENWQE_CARD_UNUSED
;
145 spin_lock_init(&cd
->print_lock
);
147 cd
->ddcb_software_timeout
= GENWQE_DDCB_SOFTWARE_TIMEOUT
;
148 cd
->kill_timeout
= GENWQE_KILL_TIMEOUT
;
150 for (j
= 0; j
< GENWQE_MAX_VFS
; j
++)
151 cd
->vf_jobtimeout_msec
[j
] = GENWQE_VF_JOBTIMEOUT_MSEC
;
153 genwqe_devices
[i
] = cd
;
157 static void genwqe_dev_free(struct genwqe_dev
*cd
)
162 genwqe_devices
[cd
->card_idx
] = NULL
;
167 * genwqe_bus_reset() - Card recovery
168 * @cd: GenWQE device information
170 * pci_reset_function() will recover the device and ensure that the
171 * registers are accessible again when it completes with success. If
172 * not, the card will stay dead and registers will be unaccessible
175 static int genwqe_bus_reset(struct genwqe_dev
*cd
)
178 struct pci_dev
*pci_dev
= cd
->pci_dev
;
181 if (cd
->err_inject
& GENWQE_INJECT_BUS_RESET_FAILURE
)
186 pci_iounmap(pci_dev
, mmio
);
188 pci_release_mem_regions(pci_dev
);
191 * Firmware/BIOS might change memory mapping during bus reset.
192 * Settings like enable bus-mastering, ... are backuped and
193 * restored by the pci_reset_function().
195 dev_dbg(&pci_dev
->dev
, "[%s] pci_reset function ...\n", __func__
);
196 rc
= pci_reset_function(pci_dev
);
198 dev_err(&pci_dev
->dev
,
199 "[%s] err: failed reset func (rc %d)\n", __func__
, rc
);
202 dev_dbg(&pci_dev
->dev
, "[%s] done with rc=%d\n", __func__
, rc
);
205 * Here is the right spot to clear the register read
206 * failure. pci_bus_reset() does this job in real systems.
208 cd
->err_inject
&= ~(GENWQE_INJECT_HARDWARE_FAILURE
|
209 GENWQE_INJECT_GFIR_FATAL
|
210 GENWQE_INJECT_GFIR_INFO
);
212 rc
= pci_request_mem_regions(pci_dev
, genwqe_driver_name
);
214 dev_err(&pci_dev
->dev
,
215 "[%s] err: request bars failed (%d)\n", __func__
, rc
);
219 cd
->mmio
= pci_iomap(pci_dev
, 0, 0);
220 if (cd
->mmio
== NULL
) {
221 dev_err(&pci_dev
->dev
,
222 "[%s] err: mapping BAR0 failed\n", __func__
);
229 * Hardware circumvention section. Certain bitstreams in our test-lab
230 * had different kinds of problems. Here is where we adjust those
231 * bitstreams to function will with this version of our device driver.
233 * Thise circumventions are applied to the physical function only.
234 * The magical numbers below are identifying development/manufacturing
235 * versions of the bitstream used on the card.
237 * Turn off error reporting for old/manufacturing images.
240 bool genwqe_need_err_masking(struct genwqe_dev
*cd
)
242 return (cd
->slu_unitcfg
& 0xFFFF0ull
) < 0x32170ull
;
245 static void genwqe_tweak_hardware(struct genwqe_dev
*cd
)
247 struct pci_dev
*pci_dev
= cd
->pci_dev
;
249 /* Mask FIRs for development images */
250 if (((cd
->slu_unitcfg
& 0xFFFF0ull
) >= 0x32000ull
) &&
251 ((cd
->slu_unitcfg
& 0xFFFF0ull
) <= 0x33250ull
)) {
252 dev_warn(&pci_dev
->dev
,
253 "FIRs masked due to bitstream %016llx.%016llx\n",
254 cd
->slu_unitcfg
, cd
->app_unitcfg
);
256 __genwqe_writeq(cd
, IO_APP_SEC_LEM_DEBUG_OVR
,
257 0xFFFFFFFFFFFFFFFFull
);
259 __genwqe_writeq(cd
, IO_APP_ERR_ACT_MASK
,
260 0x0000000000000000ull
);
265 * genwqe_recovery_on_fatal_gfir_required() - Version depended actions
266 * @cd: GenWQE device information
268 * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must
269 * be ignored. This is e.g. true for the bitstream we gave to the card
270 * manufacturer, but also for some old bitstreams we released to our
273 int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev
*cd
)
275 return (cd
->slu_unitcfg
& 0xFFFF0ull
) >= 0x32170ull
;
278 int genwqe_flash_readback_fails(struct genwqe_dev
*cd
)
280 return (cd
->slu_unitcfg
& 0xFFFF0ull
) < 0x32170ull
;
284 * genwqe_T_psec() - Calculate PF/VF timeout register content
285 * @cd: GenWQE device information
287 * Note: From a design perspective it turned out to be a bad idea to
288 * use codes here to specifiy the frequency/speed values. An old
289 * driver cannot understand new codes and is therefore always a
290 * problem. Better is to measure out the value or put the
291 * speed/frequency directly into a register which is always a valid
292 * value for old as well as for new software.
295 static int genwqe_T_psec(struct genwqe_dev
*cd
)
297 u16 speed
; /* 1/f -> 250, 200, 166, 175 */
298 static const int T
[] = { 4000, 5000, 6000, 5714 };
300 speed
= (u16
)((cd
->slu_unitcfg
>> 28) & 0x0full
);
301 if (speed
>= ARRAY_SIZE(T
))
302 return -1; /* illegal value */
308 * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution
309 * @cd: GenWQE device information
311 * Do this _after_ card_reset() is called. Otherwise the values will
312 * vanish. The settings need to be done when the queues are inactive.
314 * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.
315 * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.
317 static bool genwqe_setup_pf_jtimer(struct genwqe_dev
*cd
)
319 u32 T
= genwqe_T_psec(cd
);
322 if (GENWQE_PF_JOBTIMEOUT_MSEC
== 0)
325 /* PF: large value needed, flash update 2sec per block */
326 x
= ilog2(GENWQE_PF_JOBTIMEOUT_MSEC
*
327 16000000000uL/(T
* 15)) - 10;
329 genwqe_write_vreg(cd
, IO_SLC_VF_APPJOB_TIMEOUT
,
330 0xff00 | (x
& 0xff), 0);
335 * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution
336 * @cd: GenWQE device information
338 static bool genwqe_setup_vf_jtimer(struct genwqe_dev
*cd
)
340 struct pci_dev
*pci_dev
= cd
->pci_dev
;
342 u32 T
= genwqe_T_psec(cd
);
346 totalvfs
= pci_sriov_get_totalvfs(pci_dev
);
350 for (vf
= 0; vf
< totalvfs
; vf
++) {
352 if (cd
->vf_jobtimeout_msec
[vf
] == 0)
355 x
= ilog2(cd
->vf_jobtimeout_msec
[vf
] *
356 16000000000uL/(T
* 15)) - 10;
358 genwqe_write_vreg(cd
, IO_SLC_VF_APPJOB_TIMEOUT
,
359 0xff00 | (x
& 0xff), vf
+ 1);
364 static int genwqe_ffdc_buffs_alloc(struct genwqe_dev
*cd
)
366 unsigned int type
, e
= 0;
368 for (type
= 0; type
< GENWQE_DBG_UNITS
; type
++) {
370 case GENWQE_DBG_UNIT0
:
371 e
= genwqe_ffdc_buff_size(cd
, 0);
373 case GENWQE_DBG_UNIT1
:
374 e
= genwqe_ffdc_buff_size(cd
, 1);
376 case GENWQE_DBG_UNIT2
:
377 e
= genwqe_ffdc_buff_size(cd
, 2);
379 case GENWQE_DBG_REGS
:
380 e
= GENWQE_FFDC_REGS
;
384 /* currently support only the debug units mentioned here */
385 cd
->ffdc
[type
].entries
= e
;
386 cd
->ffdc
[type
].regs
=
387 kmalloc_array(e
, sizeof(struct genwqe_reg
),
390 * regs == NULL is ok, the using code treats this as no regs,
391 * Printing warning is ok in this case.
397 static void genwqe_ffdc_buffs_free(struct genwqe_dev
*cd
)
401 for (type
= 0; type
< GENWQE_DBG_UNITS
; type
++) {
402 kfree(cd
->ffdc
[type
].regs
);
403 cd
->ffdc
[type
].regs
= NULL
;
407 static int genwqe_read_ids(struct genwqe_dev
*cd
)
411 struct pci_dev
*pci_dev
= cd
->pci_dev
;
413 cd
->slu_unitcfg
= __genwqe_readq(cd
, IO_SLU_UNITCFG
);
414 if (cd
->slu_unitcfg
== IO_ILLEGAL_VALUE
) {
415 dev_err(&pci_dev
->dev
,
416 "err: SLUID=%016llx\n", cd
->slu_unitcfg
);
421 slu_id
= genwqe_get_slu_id(cd
);
422 if (slu_id
< GENWQE_SLU_ARCH_REQ
|| slu_id
== 0xff) {
423 dev_err(&pci_dev
->dev
,
424 "err: incompatible SLU Architecture %u\n", slu_id
);
429 cd
->app_unitcfg
= __genwqe_readq(cd
, IO_APP_UNITCFG
);
430 if (cd
->app_unitcfg
== IO_ILLEGAL_VALUE
) {
431 dev_err(&pci_dev
->dev
,
432 "err: APPID=%016llx\n", cd
->app_unitcfg
);
436 genwqe_read_app_id(cd
, cd
->app_name
, sizeof(cd
->app_name
));
439 * Is access to all registers possible? If we are a VF the
440 * answer is obvious. If we run fully virtualized, we need to
441 * check if we can access all registers. If we do not have
442 * full access we will cause an UR and some informational FIRs
443 * in the PF, but that should not harm.
445 if (pci_dev
->is_virtfn
)
446 cd
->is_privileged
= 0;
448 cd
->is_privileged
= (__genwqe_readq(cd
, IO_SLU_BITSTREAM
)
449 != IO_ILLEGAL_VALUE
);
455 static int genwqe_start(struct genwqe_dev
*cd
)
458 struct pci_dev
*pci_dev
= cd
->pci_dev
;
460 err
= genwqe_read_ids(cd
);
464 if (genwqe_is_privileged(cd
)) {
465 /* do this after the tweaks. alloc fail is acceptable */
466 genwqe_ffdc_buffs_alloc(cd
);
467 genwqe_stop_traps(cd
);
469 /* Collect registers e.g. FIRs, UNITIDs, traces ... */
470 genwqe_read_ffdc_regs(cd
, cd
->ffdc
[GENWQE_DBG_REGS
].regs
,
471 cd
->ffdc
[GENWQE_DBG_REGS
].entries
, 0);
473 genwqe_ffdc_buff_read(cd
, GENWQE_DBG_UNIT0
,
474 cd
->ffdc
[GENWQE_DBG_UNIT0
].regs
,
475 cd
->ffdc
[GENWQE_DBG_UNIT0
].entries
);
477 genwqe_ffdc_buff_read(cd
, GENWQE_DBG_UNIT1
,
478 cd
->ffdc
[GENWQE_DBG_UNIT1
].regs
,
479 cd
->ffdc
[GENWQE_DBG_UNIT1
].entries
);
481 genwqe_ffdc_buff_read(cd
, GENWQE_DBG_UNIT2
,
482 cd
->ffdc
[GENWQE_DBG_UNIT2
].regs
,
483 cd
->ffdc
[GENWQE_DBG_UNIT2
].entries
);
485 genwqe_start_traps(cd
);
487 if (cd
->card_state
== GENWQE_CARD_FATAL_ERROR
) {
488 dev_warn(&pci_dev
->dev
,
489 "[%s] chip reload/recovery!\n", __func__
);
492 * Stealth Mode: Reload chip on either hot
495 cd
->softreset
= 0x7Cull
;
496 __genwqe_writeq(cd
, IO_SLC_CFGREG_SOFTRESET
,
499 err
= genwqe_bus_reset(cd
);
501 dev_err(&pci_dev
->dev
,
502 "[%s] err: bus reset failed!\n",
508 * Re-read the IDs because
509 * it could happen that the bitstream load
512 err
= genwqe_read_ids(cd
);
518 err
= genwqe_setup_service_layer(cd
); /* does a reset to the card */
520 dev_err(&pci_dev
->dev
,
521 "[%s] err: could not setup servicelayer!\n", __func__
);
526 if (genwqe_is_privileged(cd
)) { /* code is running _after_ reset */
527 genwqe_tweak_hardware(cd
);
529 genwqe_setup_pf_jtimer(cd
);
530 genwqe_setup_vf_jtimer(cd
);
533 err
= genwqe_device_create(cd
);
535 dev_err(&pci_dev
->dev
,
536 "err: chdev init failed! (err=%d)\n", err
);
537 goto out_release_service_layer
;
541 out_release_service_layer
:
542 genwqe_release_service_layer(cd
);
544 if (genwqe_is_privileged(cd
))
545 genwqe_ffdc_buffs_free(cd
);
550 * genwqe_stop() - Stop card operation
551 * @cd: GenWQE device information
554 * As long as genwqe_thread runs we might access registers during
555 * error data capture. Same is with the genwqe_health_thread.
556 * When genwqe_bus_reset() fails this function might called two times:
557 * first by the genwqe_health_thread() and later by genwqe_remove() to
558 * unbind the device. We must be able to survive that.
560 * This function must be robust enough to be called twice.
562 static int genwqe_stop(struct genwqe_dev
*cd
)
564 genwqe_finish_queue(cd
); /* no register access */
565 genwqe_device_remove(cd
); /* device removed, procs killed */
566 genwqe_release_service_layer(cd
); /* here genwqe_thread is stopped */
568 if (genwqe_is_privileged(cd
)) {
569 pci_disable_sriov(cd
->pci_dev
); /* access pci config space */
570 genwqe_ffdc_buffs_free(cd
);
577 * genwqe_recover_card() - Try to recover the card if it is possible
578 * @cd: GenWQE device information
579 * @fatal_err: Indicate whether to attempt soft reset
581 * If fatal_err is set no register access is possible anymore. It is
582 * likely that genwqe_start fails in that situation. Proper error
583 * handling is required in this case.
585 * genwqe_bus_reset() will cause the pci code to call genwqe_remove()
586 * and later genwqe_probe() for all virtual functions.
588 static int genwqe_recover_card(struct genwqe_dev
*cd
, int fatal_err
)
591 struct pci_dev
*pci_dev
= cd
->pci_dev
;
596 * Make sure chip is not reloaded to maintain FFDC. Write SLU
597 * Reset Register, CPLDReset field to 0.
600 cd
->softreset
= 0x70ull
;
601 __genwqe_writeq(cd
, IO_SLC_CFGREG_SOFTRESET
, cd
->softreset
);
604 rc
= genwqe_bus_reset(cd
);
606 dev_err(&pci_dev
->dev
,
607 "[%s] err: card recovery impossible!\n", __func__
);
611 rc
= genwqe_start(cd
);
613 dev_err(&pci_dev
->dev
,
614 "[%s] err: failed to launch device!\n", __func__
);
620 static int genwqe_health_check_cond(struct genwqe_dev
*cd
, u64
*gfir
)
622 *gfir
= __genwqe_readq(cd
, IO_SLC_CFGREG_GFIR
);
623 return (*gfir
& GFIR_ERR_TRIGGER
) &&
624 genwqe_recovery_on_fatal_gfir_required(cd
);
628 * genwqe_fir_checking() - Check the fault isolation registers of the card
629 * @cd: GenWQE device information
631 * If this code works ok, can be tried out with help of the genwqe_poke tool:
632 * sudo ./tools/genwqe_poke 0x8 0xfefefefefef
634 * Now the relevant FIRs/sFIRs should be printed out and the driver should
635 * invoke recovery (devices are removed and readded).
637 static u64
genwqe_fir_checking(struct genwqe_dev
*cd
)
639 int j
, iterations
= 0;
640 u64 mask
, fir
, fec
, uid
, gfir
, gfir_masked
, sfir
, sfec
;
641 u32 fir_addr
, fir_clr_addr
, fec_addr
, sfir_addr
, sfec_addr
;
642 struct pci_dev
*pci_dev
= cd
->pci_dev
;
646 if (iterations
> 16) {
647 dev_err(&pci_dev
->dev
, "* exit looping after %d times\n",
652 gfir
= __genwqe_readq(cd
, IO_SLC_CFGREG_GFIR
);
654 dev_err(&pci_dev
->dev
, "* 0x%08x 0x%016llx\n",
655 IO_SLC_CFGREG_GFIR
, gfir
);
656 if (gfir
== IO_ILLEGAL_VALUE
)
660 * Avoid printing when to GFIR bit is on prevents contignous
661 * printout e.g. for the following bug:
662 * FIR set without a 2ndary FIR/FIR cannot be cleared
663 * Comment out the following if to get the prints:
668 gfir_masked
= gfir
& GFIR_ERR_TRIGGER
; /* fatal errors */
670 for (uid
= 0; uid
< GENWQE_MAX_UNITS
; uid
++) { /* 0..2 in zEDC */
672 /* read the primary FIR (pfir) */
673 fir_addr
= (uid
<< 24) + 0x08;
674 fir
= __genwqe_readq(cd
, fir_addr
);
676 continue; /* no error in this unit */
678 dev_err(&pci_dev
->dev
, "* 0x%08x 0x%016llx\n", fir_addr
, fir
);
679 if (fir
== IO_ILLEGAL_VALUE
)
682 /* read primary FEC */
683 fec_addr
= (uid
<< 24) + 0x18;
684 fec
= __genwqe_readq(cd
, fec_addr
);
686 dev_err(&pci_dev
->dev
, "* 0x%08x 0x%016llx\n", fec_addr
, fec
);
687 if (fec
== IO_ILLEGAL_VALUE
)
690 for (j
= 0, mask
= 1ULL; j
< 64; j
++, mask
<<= 1) {
692 /* secondary fir empty, skip it */
693 if ((fir
& mask
) == 0x0)
696 sfir_addr
= (uid
<< 24) + 0x100 + 0x08 * j
;
697 sfir
= __genwqe_readq(cd
, sfir_addr
);
699 if (sfir
== IO_ILLEGAL_VALUE
)
701 dev_err(&pci_dev
->dev
,
702 "* 0x%08x 0x%016llx\n", sfir_addr
, sfir
);
704 sfec_addr
= (uid
<< 24) + 0x300 + 0x08 * j
;
705 sfec
= __genwqe_readq(cd
, sfec_addr
);
707 if (sfec
== IO_ILLEGAL_VALUE
)
709 dev_err(&pci_dev
->dev
,
710 "* 0x%08x 0x%016llx\n", sfec_addr
, sfec
);
712 gfir
= __genwqe_readq(cd
, IO_SLC_CFGREG_GFIR
);
713 if (gfir
== IO_ILLEGAL_VALUE
)
716 /* gfir turned on during routine! get out and
718 if ((gfir_masked
== 0x0) &&
719 (gfir
& GFIR_ERR_TRIGGER
)) {
723 /* do not clear if we entered with a fatal gfir */
724 if (gfir_masked
== 0x0) {
726 /* NEW clear by mask the logged bits */
727 sfir_addr
= (uid
<< 24) + 0x100 + 0x08 * j
;
728 __genwqe_writeq(cd
, sfir_addr
, sfir
);
730 dev_dbg(&pci_dev
->dev
,
731 "[HM] Clearing 2ndary FIR 0x%08x with 0x%016llx\n",
735 * note, these cannot be error-Firs
736 * since gfir_masked is 0 after sfir
737 * was read. Also, it is safe to do
738 * this write if sfir=0. Still need to
739 * clear the primary. This just means
740 * there is no secondary FIR.
743 /* clear by mask the logged bit. */
744 fir_clr_addr
= (uid
<< 24) + 0x10;
745 __genwqe_writeq(cd
, fir_clr_addr
, mask
);
747 dev_dbg(&pci_dev
->dev
,
748 "[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",
753 gfir
= __genwqe_readq(cd
, IO_SLC_CFGREG_GFIR
);
754 if (gfir
== IO_ILLEGAL_VALUE
)
757 if ((gfir_masked
== 0x0) && (gfir
& GFIR_ERR_TRIGGER
)) {
759 * Check once more that it didn't go on after all the
762 dev_dbg(&pci_dev
->dev
, "ACK! Another FIR! Recursing %d!\n",
769 return IO_ILLEGAL_VALUE
;
773 * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot
774 * @pci_dev: PCI device information struct
776 * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this
777 * reset method will not work in all cases.
779 * Return: 0 on success or error code from pci_set_pcie_reset_state()
781 static int genwqe_pci_fundamental_reset(struct pci_dev
*pci_dev
)
786 * lock pci config space access from userspace,
787 * save state and issue PCIe fundamental reset
789 pci_cfg_access_lock(pci_dev
);
790 pci_save_state(pci_dev
);
791 rc
= pci_set_pcie_reset_state(pci_dev
, pcie_warm_reset
);
793 /* keep PCIe reset asserted for 250ms */
795 pci_set_pcie_reset_state(pci_dev
, pcie_deassert_reset
);
796 /* Wait for 2s to reload flash and train the link */
799 pci_restore_state(pci_dev
);
800 pci_cfg_access_unlock(pci_dev
);
805 static int genwqe_platform_recovery(struct genwqe_dev
*cd
)
807 struct pci_dev
*pci_dev
= cd
->pci_dev
;
810 dev_info(&pci_dev
->dev
,
811 "[%s] resetting card for error recovery\n", __func__
);
813 /* Clear out error injection flags */
814 cd
->err_inject
&= ~(GENWQE_INJECT_HARDWARE_FAILURE
|
815 GENWQE_INJECT_GFIR_FATAL
|
816 GENWQE_INJECT_GFIR_INFO
);
820 /* Try recoverying the card with fundamental reset */
821 rc
= genwqe_pci_fundamental_reset(pci_dev
);
823 rc
= genwqe_start(cd
);
825 dev_info(&pci_dev
->dev
,
826 "[%s] card recovered\n", __func__
);
828 dev_err(&pci_dev
->dev
,
829 "[%s] err: cannot start card services! (err=%d)\n",
832 dev_err(&pci_dev
->dev
,
833 "[%s] card reset failed\n", __func__
);
840 * genwqe_reload_bistream() - reload card bitstream
841 * @cd: GenWQE device information
843 * Set the appropriate register and call fundamental reset to reaload the card
846 * Return: 0 on success, error code otherwise
848 static int genwqe_reload_bistream(struct genwqe_dev
*cd
)
850 struct pci_dev
*pci_dev
= cd
->pci_dev
;
853 dev_info(&pci_dev
->dev
,
854 "[%s] resetting card for bitstream reload\n",
860 * Cause a CPLD reprogram with the 'next_bitstream'
861 * partition on PCIe hot or fundamental reset
863 __genwqe_writeq(cd
, IO_SLC_CFGREG_SOFTRESET
,
864 (cd
->softreset
& 0xcull
) | 0x70ull
);
866 rc
= genwqe_pci_fundamental_reset(pci_dev
);
869 * A fundamental reset failure can be caused
870 * by lack of support on the arch, so we just
871 * log the error and try to start the card
874 dev_err(&pci_dev
->dev
,
875 "[%s] err: failed to reset card for bitstream reload\n",
879 rc
= genwqe_start(cd
);
881 dev_err(&pci_dev
->dev
,
882 "[%s] err: cannot start card services! (err=%d)\n",
886 dev_info(&pci_dev
->dev
,
887 "[%s] card reloaded\n", __func__
);
893 * genwqe_health_thread() - Health checking thread
894 * @data: GenWQE device information
896 * This thread is only started for the PF of the card.
898 * This thread monitors the health of the card. A critical situation
899 * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In
900 * this case we need to be recovered from outside. Writing to
901 * registers will very likely not work either.
903 * This thread must only exit if kthread_should_stop() becomes true.
905 * Condition for the health-thread to trigger:
906 * a) when a kthread_stop() request comes in or
907 * b) a critical GFIR occured
909 * Informational GFIRs are checked and potentially printed in
910 * GENWQE_HEALTH_CHECK_INTERVAL seconds.
912 static int genwqe_health_thread(void *data
)
914 int rc
, should_stop
= 0;
915 struct genwqe_dev
*cd
= data
;
916 struct pci_dev
*pci_dev
= cd
->pci_dev
;
917 u64 gfir
, gfir_masked
, slu_unitcfg
, app_unitcfg
;
920 while (!kthread_should_stop()) {
921 rc
= wait_event_interruptible_timeout(cd
->health_waitq
,
922 (genwqe_health_check_cond(cd
, &gfir
) ||
923 (should_stop
= kthread_should_stop())),
924 GENWQE_HEALTH_CHECK_INTERVAL
* HZ
);
929 if (gfir
== IO_ILLEGAL_VALUE
) {
930 dev_err(&pci_dev
->dev
,
931 "[%s] GFIR=%016llx\n", __func__
, gfir
);
935 slu_unitcfg
= __genwqe_readq(cd
, IO_SLU_UNITCFG
);
936 if (slu_unitcfg
== IO_ILLEGAL_VALUE
) {
937 dev_err(&pci_dev
->dev
,
938 "[%s] SLU_UNITCFG=%016llx\n",
939 __func__
, slu_unitcfg
);
943 app_unitcfg
= __genwqe_readq(cd
, IO_APP_UNITCFG
);
944 if (app_unitcfg
== IO_ILLEGAL_VALUE
) {
945 dev_err(&pci_dev
->dev
,
946 "[%s] APP_UNITCFG=%016llx\n",
947 __func__
, app_unitcfg
);
951 gfir
= __genwqe_readq(cd
, IO_SLC_CFGREG_GFIR
);
952 if (gfir
== IO_ILLEGAL_VALUE
) {
953 dev_err(&pci_dev
->dev
,
954 "[%s] %s: GFIR=%016llx\n", __func__
,
955 (gfir
& GFIR_ERR_TRIGGER
) ? "err" : "info",
960 gfir_masked
= genwqe_fir_checking(cd
);
961 if (gfir_masked
== IO_ILLEGAL_VALUE
)
965 * GFIR ErrorTrigger bits set => reset the card!
966 * Never do this for old/manufacturing images!
968 if ((gfir_masked
) && !cd
->skip_recovery
&&
969 genwqe_recovery_on_fatal_gfir_required(cd
)) {
971 cd
->card_state
= GENWQE_CARD_FATAL_ERROR
;
973 rc
= genwqe_recover_card(cd
, 0);
975 /* FIXME Card is unusable and needs unbind! */
980 if (cd
->card_state
== GENWQE_CARD_RELOAD_BITSTREAM
) {
981 /* Userspace requested card bitstream reload */
982 rc
= genwqe_reload_bistream(cd
);
987 cd
->last_gfir
= gfir
;
994 if (cd
->use_platform_recovery
) {
996 * Since we use raw accessors, EEH errors won't be detected
997 * by the platform until we do a non-raw MMIO or config space
1000 readq(cd
->mmio
+ IO_SLC_CFGREG_GFIR
);
1002 /* We do nothing if the card is going over PCI recovery */
1003 if (pci_channel_offline(pci_dev
))
1007 * If it's supported by the platform, we try a fundamental reset
1008 * to recover from a fatal error. Otherwise, we continue to wait
1009 * for an external recovery procedure to take care of it.
1011 rc
= genwqe_platform_recovery(cd
);
1013 goto health_thread_begin
;
1016 dev_err(&pci_dev
->dev
,
1017 "[%s] card unusable. Please trigger unbind!\n", __func__
);
1019 /* Bring down logical devices to inform user space via udev remove. */
1020 cd
->card_state
= GENWQE_CARD_FATAL_ERROR
;
1023 /* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */
1024 while (!kthread_should_stop())
1030 static int genwqe_health_check_start(struct genwqe_dev
*cd
)
1034 if (GENWQE_HEALTH_CHECK_INTERVAL
<= 0)
1035 return 0; /* valid for disabling the service */
1037 /* moved before request_irq() */
1038 /* init_waitqueue_head(&cd->health_waitq); */
1040 cd
->health_thread
= kthread_run(genwqe_health_thread
, cd
,
1041 GENWQE_DEVNAME
"%d_health",
1043 if (IS_ERR(cd
->health_thread
)) {
1044 rc
= PTR_ERR(cd
->health_thread
);
1045 cd
->health_thread
= NULL
;
1051 static int genwqe_health_thread_running(struct genwqe_dev
*cd
)
1053 return cd
->health_thread
!= NULL
;
1056 static int genwqe_health_check_stop(struct genwqe_dev
*cd
)
1058 if (!genwqe_health_thread_running(cd
))
1061 kthread_stop(cd
->health_thread
);
1062 cd
->health_thread
= NULL
;
1067 * genwqe_pci_setup() - Allocate PCIe related resources for our card
1068 * @cd: GenWQE device information
1070 static int genwqe_pci_setup(struct genwqe_dev
*cd
)
1073 struct pci_dev
*pci_dev
= cd
->pci_dev
;
1075 err
= pci_enable_device_mem(pci_dev
);
1077 dev_err(&pci_dev
->dev
,
1078 "err: failed to enable pci memory (err=%d)\n", err
);
1082 /* Reserve PCI I/O and memory resources */
1083 err
= pci_request_mem_regions(pci_dev
, genwqe_driver_name
);
1085 dev_err(&pci_dev
->dev
,
1086 "[%s] err: request bars failed (%d)\n", __func__
, err
);
1088 goto err_disable_device
;
1091 /* check for 64-bit DMA address supported (DAC) */
1092 /* check for 32-bit DMA address supported (SAC) */
1093 if (dma_set_mask_and_coherent(&pci_dev
->dev
, DMA_BIT_MASK(64)) &&
1094 dma_set_mask_and_coherent(&pci_dev
->dev
, DMA_BIT_MASK(32))) {
1095 dev_err(&pci_dev
->dev
,
1096 "err: neither DMA32 nor DMA64 supported\n");
1098 goto out_release_resources
;
1101 pci_set_master(pci_dev
);
1102 pci_enable_pcie_error_reporting(pci_dev
);
1104 /* EEH recovery requires PCIe fundamental reset */
1105 pci_dev
->needs_freset
= 1;
1107 /* request complete BAR-0 space (length = 0) */
1108 cd
->mmio_len
= pci_resource_len(pci_dev
, 0);
1109 cd
->mmio
= pci_iomap(pci_dev
, 0, 0);
1110 if (cd
->mmio
== NULL
) {
1111 dev_err(&pci_dev
->dev
,
1112 "[%s] err: mapping BAR0 failed\n", __func__
);
1114 goto out_release_resources
;
1117 cd
->num_vfs
= pci_sriov_get_totalvfs(pci_dev
);
1118 if (cd
->num_vfs
< 0)
1121 err
= genwqe_read_ids(cd
);
1128 pci_iounmap(pci_dev
, cd
->mmio
);
1129 out_release_resources
:
1130 pci_release_mem_regions(pci_dev
);
1132 pci_disable_device(pci_dev
);
1138 * genwqe_pci_remove() - Free PCIe related resources for our card
1139 * @cd: GenWQE device information
1141 static void genwqe_pci_remove(struct genwqe_dev
*cd
)
1143 struct pci_dev
*pci_dev
= cd
->pci_dev
;
1146 pci_iounmap(pci_dev
, cd
->mmio
);
1148 pci_release_mem_regions(pci_dev
);
1149 pci_disable_device(pci_dev
);
1153 * genwqe_probe() - Device initialization
1154 * @pci_dev: PCI device information struct
1155 * @id: PCI device ID
1157 * Callable for multiple cards. This function is called on bind.
1159 * Return: 0 if succeeded, < 0 when failed
1161 static int genwqe_probe(struct pci_dev
*pci_dev
,
1162 const struct pci_device_id
*id
)
1165 struct genwqe_dev
*cd
;
1167 genwqe_init_crc32();
1169 cd
= genwqe_dev_alloc();
1171 dev_err(&pci_dev
->dev
, "err: could not alloc mem (err=%d)!\n",
1176 dev_set_drvdata(&pci_dev
->dev
, cd
);
1177 cd
->pci_dev
= pci_dev
;
1179 err
= genwqe_pci_setup(cd
);
1181 dev_err(&pci_dev
->dev
,
1182 "err: problems with PCI setup (err=%d)\n", err
);
1186 err
= genwqe_start(cd
);
1188 dev_err(&pci_dev
->dev
,
1189 "err: cannot start card services! (err=%d)\n", err
);
1190 goto out_pci_remove
;
1193 if (genwqe_is_privileged(cd
)) {
1194 err
= genwqe_health_check_start(cd
);
1196 dev_err(&pci_dev
->dev
,
1197 "err: cannot start health checking! (err=%d)\n",
1199 goto out_stop_services
;
1207 genwqe_pci_remove(cd
);
1209 genwqe_dev_free(cd
);
1214 * genwqe_remove() - Called when device is removed (hot-plugable)
1215 * @pci_dev: PCI device information struct
1217 * Or when driver is unloaded respecitively when unbind is done.
1219 static void genwqe_remove(struct pci_dev
*pci_dev
)
1221 struct genwqe_dev
*cd
= dev_get_drvdata(&pci_dev
->dev
);
1223 genwqe_health_check_stop(cd
);
1226 * genwqe_stop() must survive if it is called twice
1227 * sequentially. This happens when the health thread calls it
1228 * and fails on genwqe_bus_reset().
1231 genwqe_pci_remove(cd
);
1232 genwqe_dev_free(cd
);
1236 * genwqe_err_error_detected() - Error detection callback
1237 * @pci_dev: PCI device information struct
1238 * @state: PCI channel state
1240 * This callback is called by the PCI subsystem whenever a PCI bus
1241 * error is detected.
1243 static pci_ers_result_t
genwqe_err_error_detected(struct pci_dev
*pci_dev
,
1244 pci_channel_state_t state
)
1246 struct genwqe_dev
*cd
;
1248 dev_err(&pci_dev
->dev
, "[%s] state=%d\n", __func__
, state
);
1250 cd
= dev_get_drvdata(&pci_dev
->dev
);
1252 return PCI_ERS_RESULT_DISCONNECT
;
1255 genwqe_health_check_stop(cd
);
1259 * On permanent failure, the PCI code will call device remove
1260 * after the return of this function.
1261 * genwqe_stop() can be called twice.
1263 if (state
== pci_channel_io_perm_failure
) {
1264 return PCI_ERS_RESULT_DISCONNECT
;
1266 genwqe_pci_remove(cd
);
1267 return PCI_ERS_RESULT_NEED_RESET
;
1271 static pci_ers_result_t
genwqe_err_slot_reset(struct pci_dev
*pci_dev
)
1274 struct genwqe_dev
*cd
= dev_get_drvdata(&pci_dev
->dev
);
1276 rc
= genwqe_pci_setup(cd
);
1278 return PCI_ERS_RESULT_RECOVERED
;
1280 dev_err(&pci_dev
->dev
,
1281 "err: problems with PCI setup (err=%d)\n", rc
);
1282 return PCI_ERS_RESULT_DISCONNECT
;
1286 static pci_ers_result_t
genwqe_err_result_none(struct pci_dev
*dev
)
1288 return PCI_ERS_RESULT_NONE
;
1291 static void genwqe_err_resume(struct pci_dev
*pci_dev
)
1294 struct genwqe_dev
*cd
= dev_get_drvdata(&pci_dev
->dev
);
1296 rc
= genwqe_start(cd
);
1298 rc
= genwqe_health_check_start(cd
);
1300 dev_err(&pci_dev
->dev
,
1301 "err: cannot start health checking! (err=%d)\n",
1304 dev_err(&pci_dev
->dev
,
1305 "err: cannot start card services! (err=%d)\n", rc
);
1309 static int genwqe_sriov_configure(struct pci_dev
*dev
, int numvfs
)
1312 struct genwqe_dev
*cd
= dev_get_drvdata(&dev
->dev
);
1315 genwqe_setup_vf_jtimer(cd
);
1316 rc
= pci_enable_sriov(dev
, numvfs
);
1322 pci_disable_sriov(dev
);
1328 static const struct pci_error_handlers genwqe_err_handler
= {
1329 .error_detected
= genwqe_err_error_detected
,
1330 .mmio_enabled
= genwqe_err_result_none
,
1331 .slot_reset
= genwqe_err_slot_reset
,
1332 .resume
= genwqe_err_resume
,
1335 static struct pci_driver genwqe_driver
= {
1336 .name
= genwqe_driver_name
,
1337 .id_table
= genwqe_device_table
,
1338 .probe
= genwqe_probe
,
1339 .remove
= genwqe_remove
,
1340 .sriov_configure
= genwqe_sriov_configure
,
1341 .err_handler
= &genwqe_err_handler
,
1345 * genwqe_devnode() - Set default access mode for genwqe devices.
1346 * @dev: Pointer to device (unused)
1347 * @mode: Carrier to pass-back given mode (permissions)
1349 * Default mode should be rw for everybody. Do not change default
1352 static char *genwqe_devnode(struct device
*dev
, umode_t
*mode
)
1360 * genwqe_init_module() - Driver registration and initialization
1362 static int __init
genwqe_init_module(void)
1366 class_genwqe
= class_create(THIS_MODULE
, GENWQE_DEVNAME
);
1367 if (IS_ERR(class_genwqe
)) {
1368 pr_err("[%s] create class failed\n", __func__
);
1372 class_genwqe
->devnode
= genwqe_devnode
;
1374 debugfs_genwqe
= debugfs_create_dir(GENWQE_DEVNAME
, NULL
);
1376 rc
= pci_register_driver(&genwqe_driver
);
1378 pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__
, rc
);
1385 debugfs_remove(debugfs_genwqe
);
1386 class_destroy(class_genwqe
);
1391 * genwqe_exit_module() - Driver exit
1393 static void __exit
genwqe_exit_module(void)
1395 pci_unregister_driver(&genwqe_driver
);
1396 debugfs_remove(debugfs_genwqe
);
1397 class_destroy(class_genwqe
);
1400 module_init(genwqe_init_module
);
1401 module_exit(genwqe_exit_module
);