UBUNTU: SAUCE: x86/quirks: Add parameter to clear MSIs early on boot

BugLink: https://bugs.launchpad.net/bugs/1797990
We observed a kdump failure in x86 that was narrowed down to MSI irq
storm coming from a PCI network device. The bug manifests as a lack of
progress in the boot process of kdump kernel, and a flood of kernel
messages like:

[...]
[ 342.265294] do_IRQ: 0.155 No irq handler for vector
[ 342.266916] do_IRQ: 0.155 No irq handler for vector
[ 347.258422] do_IRQ: 14053260 callbacks suppressed
[...]

The root cause of the issue is that kexec process of the kdump kernel
doesn't ensure PCI devices are reset or MSI capabilities are disabled,
so a PCI adapter could produce a huge amount of irqs which would steal
all the processing time for the CPU (specially since we usually restrict
kdump kernel to use a single CPU only).

This patch implements the kernel parameter "pci=clearmsi" to clear the
MSI/MSI-X enable bits in the Message Control register for all PCI devices
during early boot time, thus preventing potential issues in the kexec'ed
kernel. PCI spec also supports/enforces this need (see PCI Local Bus
spec sections 6.8.1.3 and 6.8.2.3).

Suggested-by: Dan Streetman <ddstreet@canonical.com>
Suggested-by: Gavin Shan <shan.gavin@linux.alibaba.com>
Signed-off-by: Guilherme G. Piccoli <gpiccoli@canonical.com>
[mfo: backport to ubuntu-bionic:
 - update context lines in pci-direct.h and early-quirks.c]
Signed-off-by: Mauricio Faria de Oliveira <mfo@canonical.com>
Acked-by: Khalid Elmously <khalid.elmously@canonical.com>
Acked-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 51210d10d905955e4412357790b7338d82d00d08..c163c3ce2902ccf7a0538eff2b0b3361c2940202 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3049,6 +3049,12 @@
                nomsi           [MSI] If the PCI_MSI kernel config parameter is
                                enabled, this kernel boot option can be used to
                                disable the use of MSI interrupts system-wide.
+               clearmsi        [X86] Clears MSI/MSI-X enable bits early in boot
+                               time in order to avoid issues like adapters
+                               screaming irqs and preventing boot progress.
+                               Also, it enforces the PCI Local Bus spec
+                               rule that those bits should be 0 in system reset
+                               events (useful for kexec/kdump cases).
                noioapicquirk   [APIC] Disable all boot interrupt quirks.
                                Safety option to keep boot IRQs enabled. This
                                should never be necessary.

diff --git a/arch/x86/include/asm/pci-direct.h b/arch/x86/include/asm/pci-direct.h
index 3f5b5158bbd72c76b7a0ba372d0855149c558a9e..14d4aa4e1aed2199b98a3106c7ccc3b0c7c7e8e5 100644 (file)
--- a/arch/x86/include/asm/pci-direct.h
+++ b/arch/x86/include/asm/pci-direct.h
@@ -15,6 +15,7 @@ extern void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, u32 val);
 extern void write_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset, u8 val);
 extern void write_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset, u16 val);
 
+extern unsigned int pci_early_clear_msi;
 extern int early_pci_allowed(void);
 
 extern unsigned int pci_early_dump_regs;

diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index c87560e1e3ef47d7948b5f01cd8678aa2ff4e835..f7faa33414926c0f29b45f3de8fe0e3e96b66f70 100644 (file)
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -28,6 +28,37 @@
 #include <asm/irq_remapping.h>
 #include <asm/early_ioremap.h>
 
+static void __init early_pci_clear_msi(int bus, int slot, int func)
+{
+       int pos;
+       u16 ctrl;
+
+       if (likely(!pci_early_clear_msi))
+               return;
+
+       pr_info_once("Clearing MSI/MSI-X enable bits early in boot (quirk)\n");
+
+       pos = pci_early_find_cap(bus, slot, func, PCI_CAP_ID_MSI);
+       if (pos) {
+               ctrl = read_pci_config_16(bus, slot, func, pos + PCI_MSI_FLAGS);
+               ctrl &= ~PCI_MSI_FLAGS_ENABLE;
+               write_pci_config_16(bus, slot, func, pos + PCI_MSI_FLAGS, ctrl);
+
+               /* Read again to flush previous write */
+               ctrl = read_pci_config_16(bus, slot, func, pos + PCI_MSI_FLAGS);
+       }
+
+       pos = pci_early_find_cap(bus, slot, func, PCI_CAP_ID_MSIX);
+       if (pos) {
+               ctrl = read_pci_config_16(bus, slot, func, pos + PCI_MSIX_FLAGS);
+               ctrl &= ~PCI_MSIX_FLAGS_ENABLE;
+               write_pci_config_16(bus, slot, func, pos + PCI_MSIX_FLAGS, ctrl);
+
+               /* Read again to flush previous write */
+               ctrl = read_pci_config_16(bus, slot, func, pos + PCI_MSIX_FLAGS);
+       }
+}
+
 #define dev_err(msg)  pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg)
 
 static void __init fix_hypertransport_config(int num, int slot, int func)
@@ -683,6 +714,7 @@ static struct chipset early_qrk[] __initdata = {
                PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, force_disable_hpet},
        { PCI_VENDOR_ID_BROADCOM, 0x4331,
          PCI_CLASS_NETWORK_OTHER, PCI_ANY_ID, 0, apple_airport_reset},
+       { PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0, early_pci_clear_msi},
        {}
 };
 

diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c
index 563049c483a12c5fe587e463fb96bff4dde22c5a..94b156f855739b7340d3792ec7d839b21e303998 100644 (file)
--- a/arch/x86/pci/common.c
+++ b/arch/x86/pci/common.c
@@ -33,6 +33,7 @@ int noioapicreroute = 1;
 #endif
 int pcibios_last_bus = -1;
 unsigned long pirq_table_addr;
+unsigned int pci_early_clear_msi;
 const struct pci_raw_ops *__read_mostly raw_pci_ops;
 const struct pci_raw_ops *__read_mostly raw_pci_ext_ops;
 
@@ -608,6 +609,9 @@ char *__init pcibios_setup(char *str)
        } else if (!strcmp(str, "skip_isa_align")) {
                pci_probe |= PCI_CAN_SKIP_ISA_ALIGN;
                return NULL;
+       } else if (!strcmp(str, "clearmsi")) {
+               pci_early_clear_msi = 1;
+               return NULL;
        } else if (!strcmp(str, "noioapicquirk")) {
                noioapicquirk = 1;
                return NULL;