2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/delay.h>
29 module_param(forceload
, bool, 0);
31 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
36 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
40 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
44 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
47 PCI_EXP_DEVCTL_NFERE
|
50 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
55 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
60 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
64 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
65 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
66 PCI_EXP_DEVCTL_NFERE
|
69 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
74 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
79 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
83 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
84 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
85 if (dev
->error_state
== pci_channel_io_normal
)
86 status
&= ~mask
; /* Clear corresponding nonfatal bits */
88 status
&= mask
; /* Clear corresponding fatal bits */
89 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
95 int pci_cleanup_aer_correct_error_status(struct pci_dev
*dev
)
100 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
104 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, &status
);
105 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, status
);
112 static void set_device_error_reporting(struct pci_dev
*dev
, void *data
)
114 bool enable
= *((bool *)data
);
116 if (dev
->pcie_type
!= PCIE_RC_PORT
&&
117 dev
->pcie_type
!= PCIE_SW_UPSTREAM_PORT
&&
118 dev
->pcie_type
!= PCIE_SW_DOWNSTREAM_PORT
)
122 pci_enable_pcie_error_reporting(dev
);
124 pci_disable_pcie_error_reporting(dev
);
128 * set_downstream_devices_error_reporting - enable/disable the error reporting bits on the root port and its downstream ports.
129 * @dev: pointer to root port's pci_dev data structure
130 * @enable: true = enable error reporting, false = disable error reporting.
132 static void set_downstream_devices_error_reporting(struct pci_dev
*dev
,
135 set_device_error_reporting(dev
, &enable
);
136 pci_walk_bus(dev
->subordinate
, set_device_error_reporting
, &enable
);
139 static int find_device_iter(struct device
*device
, void *data
)
142 u16 id
= *(unsigned long *)data
;
143 u8 secondary
, subordinate
, d_bus
= id
>> 8;
145 if (device
->bus
== &pci_bus_type
) {
146 dev
= to_pci_dev(device
);
147 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
151 *(unsigned long*)data
= (unsigned long)device
;
156 * If device is P2P, check if it is an upstream?
158 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
159 pci_read_config_byte(dev
, PCI_SECONDARY_BUS
,
161 pci_read_config_byte(dev
, PCI_SUBORDINATE_BUS
,
163 if (d_bus
>= secondary
&& d_bus
<= subordinate
) {
164 *(unsigned long*)data
= (unsigned long)device
;
174 * find_source_device - search through device hierarchy for source device
175 * @parent: pointer to Root Port pci_dev data structure
176 * @id: device ID of agent who sends an error message to this Root Port
178 * Invoked when error is detected at the Root Port.
180 static struct device
* find_source_device(struct pci_dev
*parent
, u16 id
)
182 struct pci_dev
*dev
= parent
;
183 struct device
*device
;
184 unsigned long device_addr
;
187 /* Is Root Port an agent that sends error message? */
188 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
193 if ((status
= device_for_each_child(&dev
->dev
,
194 &device_addr
, find_device_iter
))) {
195 device
= (struct device
*)device_addr
;
196 dev
= to_pci_dev(device
);
197 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
205 static void report_error_detected(struct pci_dev
*dev
, void *data
)
207 pci_ers_result_t vote
;
208 struct pci_error_handlers
*err_handler
;
209 struct aer_broadcast_data
*result_data
;
210 result_data
= (struct aer_broadcast_data
*) data
;
212 dev
->error_state
= result_data
->state
;
215 !dev
->driver
->err_handler
||
216 !dev
->driver
->err_handler
->error_detected
) {
217 if (result_data
->state
== pci_channel_io_frozen
&&
218 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
220 * In case of fatal recovery, if one of down-
221 * stream device has no driver. We might be
222 * unable to recover because a later insmod
223 * of a driver for this device is unaware of
226 dev_printk(KERN_DEBUG
, &dev
->dev
, "device has %s\n",
228 "no AER-aware driver" : "no driver");
233 err_handler
= dev
->driver
->err_handler
;
234 vote
= err_handler
->error_detected(dev
, result_data
->state
);
235 result_data
->result
= merge_result(result_data
->result
, vote
);
239 static void report_mmio_enabled(struct pci_dev
*dev
, void *data
)
241 pci_ers_result_t vote
;
242 struct pci_error_handlers
*err_handler
;
243 struct aer_broadcast_data
*result_data
;
244 result_data
= (struct aer_broadcast_data
*) data
;
247 !dev
->driver
->err_handler
||
248 !dev
->driver
->err_handler
->mmio_enabled
)
251 err_handler
= dev
->driver
->err_handler
;
252 vote
= err_handler
->mmio_enabled(dev
);
253 result_data
->result
= merge_result(result_data
->result
, vote
);
257 static void report_slot_reset(struct pci_dev
*dev
, void *data
)
259 pci_ers_result_t vote
;
260 struct pci_error_handlers
*err_handler
;
261 struct aer_broadcast_data
*result_data
;
262 result_data
= (struct aer_broadcast_data
*) data
;
265 !dev
->driver
->err_handler
||
266 !dev
->driver
->err_handler
->slot_reset
)
269 err_handler
= dev
->driver
->err_handler
;
270 vote
= err_handler
->slot_reset(dev
);
271 result_data
->result
= merge_result(result_data
->result
, vote
);
275 static void report_resume(struct pci_dev
*dev
, void *data
)
277 struct pci_error_handlers
*err_handler
;
279 dev
->error_state
= pci_channel_io_normal
;
282 !dev
->driver
->err_handler
||
283 !dev
->driver
->err_handler
->resume
)
286 err_handler
= dev
->driver
->err_handler
;
287 err_handler
->resume(dev
);
292 * broadcast_error_message - handle message broadcast to downstream drivers
293 * @dev: pointer to from where in a hierarchy message is broadcasted down
294 * @state: error state
295 * @error_mesg: message to print
296 * @cb: callback to be broadcasted
298 * Invoked during error recovery process. Once being invoked, the content
299 * of error severity will be broadcasted to all downstream drivers in a
300 * hierarchy in question.
302 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
303 enum pci_channel_state state
,
305 void (*cb
)(struct pci_dev
*, void *))
307 struct aer_broadcast_data result_data
;
309 dev_printk(KERN_DEBUG
, &dev
->dev
, "broadcast %s message\n", error_mesg
);
310 result_data
.state
= state
;
311 if (cb
== report_error_detected
)
312 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
314 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
316 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
318 * If the error is reported by a bridge, we think this error
319 * is related to the downstream link of the bridge, so we
320 * do error recovery on all subordinates of the bridge instead
321 * of the bridge and clear the error status of the bridge.
323 if (cb
== report_error_detected
)
324 dev
->error_state
= state
;
325 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
326 if (cb
== report_resume
) {
327 pci_cleanup_aer_uncorrect_error_status(dev
);
328 dev
->error_state
= pci_channel_io_normal
;
333 * If the error is reported by an end point, we think this
334 * error is related to the upstream link of the end point.
336 pci_walk_bus(dev
->bus
, cb
, &result_data
);
339 return result_data
.result
;
342 struct find_aer_service_data
{
343 struct pcie_port_service_driver
*aer_driver
;
347 static int find_aer_service_iter(struct device
*device
, void *data
)
349 struct device_driver
*driver
;
350 struct pcie_port_service_driver
*service_driver
;
351 struct pcie_device
*pcie_dev
;
352 struct find_aer_service_data
*result
;
354 result
= (struct find_aer_service_data
*) data
;
356 if (device
->bus
== &pcie_port_bus_type
) {
357 pcie_dev
= to_pcie_device(device
);
358 if (pcie_dev
->id
.port_type
== PCIE_SW_DOWNSTREAM_PORT
)
359 result
->is_downstream
= 1;
361 driver
= device
->driver
;
363 service_driver
= to_service_driver(driver
);
364 if (service_driver
->id_table
->service_type
==
365 PCIE_PORT_SERVICE_AER
) {
366 result
->aer_driver
= service_driver
;
375 static void find_aer_service(struct pci_dev
*dev
,
376 struct find_aer_service_data
*data
)
379 retval
= device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
382 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
385 struct pci_dev
*udev
;
386 pci_ers_result_t status
;
387 struct find_aer_service_data data
;
389 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
392 udev
= dev
->bus
->self
;
394 data
.is_downstream
= 0;
395 data
.aer_driver
= NULL
;
396 find_aer_service(udev
, &data
);
399 * Use the aer driver of the error agent firstly.
400 * If it hasn't the aer driver, use the root port's
402 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
403 if (data
.is_downstream
&&
404 aerdev
->device
.driver
&&
405 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
407 to_service_driver(aerdev
->device
.driver
);
409 dev_printk(KERN_DEBUG
, &dev
->dev
, "no link-reset "
411 return PCI_ERS_RESULT_DISCONNECT
;
415 status
= data
.aer_driver
->reset_link(udev
);
416 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
417 dev_printk(KERN_DEBUG
, &dev
->dev
, "link reset at upstream "
418 "device %s failed\n", pci_name(udev
));
419 return PCI_ERS_RESULT_DISCONNECT
;
426 * do_recovery - handle nonfatal/fatal error recovery process
427 * @aerdev: pointer to a pcie_device data structure of root port
428 * @dev: pointer to a pci_dev data structure of agent detecting an error
429 * @severity: error severity type
431 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
432 * error detected message to all downstream drivers within a hierarchy in
433 * question and return the returned code.
435 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
439 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
440 enum pci_channel_state state
;
442 if (severity
== AER_FATAL
)
443 state
= pci_channel_io_frozen
;
445 state
= pci_channel_io_normal
;
447 status
= broadcast_error_message(dev
,
450 report_error_detected
);
452 if (severity
== AER_FATAL
) {
453 result
= reset_link(aerdev
, dev
);
454 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
455 /* TODO: Should panic here? */
460 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
461 status
= broadcast_error_message(dev
,
464 report_mmio_enabled
);
466 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
468 * TODO: Should call platform-specific
469 * functions to reset slot before calling
470 * drivers' slot_reset callbacks?
472 status
= broadcast_error_message(dev
,
478 if (status
== PCI_ERS_RESULT_RECOVERED
)
479 broadcast_error_message(dev
,
488 * handle_error_source - handle logging error into an event log
489 * @aerdev: pointer to pcie_device data structure of the root port
490 * @dev: pointer to pci_dev data structure of error source device
491 * @info: comprehensive error information
493 * Invoked when an error being detected by Root Port.
495 static void handle_error_source(struct pcie_device
* aerdev
,
497 struct aer_err_info info
)
499 pci_ers_result_t status
= 0;
502 if (info
.severity
== AER_CORRECTABLE
) {
504 * Correctable error does not need software intevention.
505 * No need to go through error recovery process.
507 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
509 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
512 status
= do_recovery(aerdev
, dev
, info
.severity
);
513 if (status
== PCI_ERS_RESULT_RECOVERED
) {
514 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver "
515 "successfully recovered\n");
517 /* TODO: Should kernel panic here? */
518 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver didn't "
525 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
526 * @rpc: pointer to a Root Port data structure
528 * Invoked when PCIE bus loads AER service driver.
530 void aer_enable_rootport(struct aer_rpc
*rpc
)
532 struct pci_dev
*pdev
= rpc
->rpd
->port
;
537 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
538 /* Clear PCIE Capability's Device Status */
539 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, ®16
);
540 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, reg16
);
542 /* Disable system error generation in response to error messages */
543 pci_read_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, ®16
);
544 reg16
&= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK
);
545 pci_write_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, reg16
);
547 aer_pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
548 /* Clear error status */
549 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
550 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
551 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, ®32
);
552 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, reg32
);
553 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, ®32
);
554 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, reg32
);
557 * Enable error reporting for the root port device and downstream port
560 set_downstream_devices_error_reporting(pdev
, true);
562 /* Enable Root Port's interrupt in response to error messages */
563 pci_write_config_dword(pdev
,
564 aer_pos
+ PCI_ERR_ROOT_COMMAND
,
565 ROOT_PORT_INTR_ON_MESG_MASK
);
569 * disable_root_aer - disable Root Port's interrupts when receiving messages
570 * @rpc: pointer to a Root Port data structure
572 * Invoked when PCIE bus unloads AER service driver.
574 static void disable_root_aer(struct aer_rpc
*rpc
)
576 struct pci_dev
*pdev
= rpc
->rpd
->port
;
581 * Disable error reporting for the root port device and downstream port
584 set_downstream_devices_error_reporting(pdev
, false);
586 pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
587 /* Disable Root's interrupt in response to error messages */
588 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_COMMAND
, 0);
590 /* Clear Root's error status reg */
591 pci_read_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
592 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
596 * get_e_source - retrieve an error source
597 * @rpc: pointer to the root port which holds an error
599 * Invoked by DPC handler to consume an error.
601 static struct aer_err_source
* get_e_source(struct aer_rpc
*rpc
)
603 struct aer_err_source
*e_source
;
606 /* Lock access to Root error producer/consumer index */
607 spin_lock_irqsave(&rpc
->e_lock
, flags
);
608 if (rpc
->prod_idx
== rpc
->cons_idx
) {
609 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
612 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
614 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
616 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
621 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
625 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
627 /* The device might not support AER */
631 if (info
->severity
== AER_CORRECTABLE
) {
632 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
634 if (!(info
->status
& ERR_CORRECTABLE_ERROR_MASK
))
635 return AER_UNSUCCESS
;
636 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
637 info
->severity
== AER_NONFATAL
) {
639 /* Link is still healthy for IO reads */
640 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
642 if (!(info
->status
& ERR_UNCORRECTABLE_ERROR_MASK
))
643 return AER_UNSUCCESS
;
645 if (info
->status
& AER_LOG_TLP_MASKS
) {
646 info
->flags
|= AER_TLP_HEADER_VALID_FLAG
;
647 pci_read_config_dword(dev
,
648 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
649 pci_read_config_dword(dev
,
650 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
651 pci_read_config_dword(dev
,
652 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
653 pci_read_config_dword(dev
,
654 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
662 * aer_isr_one_error - consume an error detected by root port
663 * @p_device: pointer to error root port service device
664 * @e_src: pointer to an error source
666 static void aer_isr_one_error(struct pcie_device
*p_device
,
667 struct aer_err_source
*e_src
)
669 struct device
*s_device
;
670 struct aer_err_info e_info
= {0, 0, 0,};
675 * There is a possibility that both correctable error and
676 * uncorrectable error being logged. Report correctable error first.
678 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
681 if (!(e_src
->status
& i
))
684 /* Init comprehensive error information */
685 if (i
& PCI_ERR_ROOT_COR_RCV
) {
686 id
= ERR_COR_ID(e_src
->id
);
687 e_info
.severity
= AER_CORRECTABLE
;
689 id
= ERR_UNCOR_ID(e_src
->id
);
690 e_info
.severity
= ((e_src
->status
>> 6) & 1);
693 (PCI_ERR_ROOT_MULTI_COR_RCV
|
694 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
695 e_info
.flags
|= AER_MULTI_ERROR_VALID_FLAG
;
696 if (!(s_device
= find_source_device(p_device
->port
, id
))) {
697 printk(KERN_DEBUG
"%s->can't find device of ID%04x\n",
701 if (get_device_error_info(to_pci_dev(s_device
), &e_info
) ==
703 aer_print_error(to_pci_dev(s_device
), &e_info
);
704 handle_error_source(p_device
,
705 to_pci_dev(s_device
),
712 * aer_isr - consume errors detected by root port
713 * @work: definition of this work item
715 * Invoked, as DPC, when root port records new detected error
717 void aer_isr(struct work_struct
*work
)
719 struct aer_rpc
*rpc
= container_of(work
, struct aer_rpc
, dpc_handler
);
720 struct pcie_device
*p_device
= rpc
->rpd
;
721 struct aer_err_source
*e_src
;
723 mutex_lock(&rpc
->rpc_mutex
);
724 e_src
= get_e_source(rpc
);
726 aer_isr_one_error(p_device
, e_src
);
727 e_src
= get_e_source(rpc
);
729 mutex_unlock(&rpc
->rpc_mutex
);
731 wake_up(&rpc
->wait_release
);
735 * aer_delete_rootport - disable root port aer and delete service data
736 * @rpc: pointer to a root port device being deleted
738 * Invoked when AER service unloaded on a specific Root Port
740 void aer_delete_rootport(struct aer_rpc
*rpc
)
742 /* Disable root port AER itself */
743 disable_root_aer(rpc
);
749 * aer_init - provide AER initialization
750 * @dev: pointer to AER pcie device
752 * Invoked when AER service driver is loaded.
754 int aer_init(struct pcie_device
*dev
)
756 if (aer_osc_setup(dev
) && !forceload
)
762 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
763 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
764 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);