patches/kernel/0020-x86-entry-64-Refactor-IRQ-stacks-and-make-them-NMI-s.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Andy Lutomirski <luto@kernel.org>
   3 Date: Tue, 11 Jul 2017 10:33:38 -0500
   4 Subject: [PATCH] x86/entry/64: Refactor IRQ stacks and make them NMI-safe
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 CVE-2017-5754
  10
  11 This will allow IRQ stacks to nest inside NMIs or similar entries
  12 that can happen during IRQ stack setup or teardown.
  13
  14 The new macros won't work correctly if they're invoked with IRQs on.
  15 Add a check under CONFIG_DEBUG_ENTRY to detect that.
  16
  17 Signed-off-by: Andy Lutomirski <luto@kernel.org>
  18 [ Use %r10 instead of %r11 in xen_do_hypervisor_callback to make objtool
  19   and ORC unwinder's lives a little easier. ]
  20 Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
  21 Cc: Borislav Petkov <bp@alien8.de>
  22 Cc: Brian Gerst <brgerst@gmail.com>
  23 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  24 Cc: H. Peter Anvin <hpa@zytor.com>
  25 Cc: Jiri Slaby <jslaby@suse.cz>
  26 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  27 Cc: Mike Galbraith <efault@gmx.de>
  28 Cc: Peter Zijlstra <peterz@infradead.org>
  29 Cc: Thomas Gleixner <tglx@linutronix.de>
  30 Cc: live-patching@vger.kernel.org
  31 Link: http://lkml.kernel.org/r/b0b2ff5fb97d2da2e1d7e1f380190c92545c8bb5.1499786555.git.jpoimboe@redhat.com
  32 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  33 (cherry picked from commit 1d3e53e8624a3ec85f4041ca6d973da7c1575938)
  34 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  35 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  36 (cherry picked from commit be58b042e135d0ee777a54798f33015857d7e2e0)
  37 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  38 ---
  39  arch/x86/kernel/process_64.c |  3 ++
  40  arch/x86/Kconfig.debug       |  2 --
  41  arch/x86/entry/entry_64.S    | 85 +++++++++++++++++++++++++++++++-------------
  42  3 files changed, 64 insertions(+), 26 deletions(-)
  43
  44 diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
  45 index fe56e6f93cbb..1e7701c4cd80 100644
  46 --- a/arch/x86/kernel/process_64.c
  47 +++ b/arch/x86/kernel/process_64.c
  48 @@ -404,6 +404,9 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
  49         int cpu = smp_processor_id();
  50         struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
  51
  52 +       WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
  53 +                    this_cpu_read(irq_count) != -1);
  54 +
  55         switch_fpu_prepare(prev_fpu, cpu);
  56
  57         /* We must save %fs and %gs before load_TLS() because
  58 diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
  59 index cd20ca0b4043..1fc519f3c49e 100644
  60 --- a/arch/x86/Kconfig.debug
  61 +++ b/arch/x86/Kconfig.debug
  62 @@ -305,8 +305,6 @@ config DEBUG_ENTRY
  63           Some of these sanity checks may slow down kernel entries and
  64           exits or otherwise impact performance.
  65
  66 -         This is currently used to help test NMI code.
  67 -
  68           If unsure, say N.
  69
  70  config DEBUG_NMI_SELFTEST
  71 diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
  72 index 6d078b89a5e8..07b4056af8a8 100644
  73 --- a/arch/x86/entry/entry_64.S
  74 +++ b/arch/x86/entry/entry_64.S
  75 @@ -447,6 +447,59 @@ ENTRY(irq_entries_start)
  76      .endr
  77  END(irq_entries_start)
  78
  79 +.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
  80 +#ifdef CONFIG_DEBUG_ENTRY
  81 +       pushfq
  82 +       testl $X86_EFLAGS_IF, (%rsp)
  83 +       jz .Lokay_\@
  84 +       ud2
  85 +.Lokay_\@:
  86 +       addq $8, %rsp
  87 +#endif
  88 +.endm
  89 +
  90 +/*
  91 + * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
  92 + * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
  93 + * Requires kernel GSBASE.
  94 + *
  95 + * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
  96 + */
  97 +.macro ENTER_IRQ_STACK old_rsp
  98 +       DEBUG_ENTRY_ASSERT_IRQS_OFF
  99 +       movq    %rsp, \old_rsp
 100 +       incl    PER_CPU_VAR(irq_count)
 101 +
 102 +       /*
 103 +        * Right now, if we just incremented irq_count to zero, we've
 104 +        * claimed the IRQ stack but we haven't switched to it yet.
 105 +        *
 106 +        * If anything is added that can interrupt us here without using IST,
 107 +        * it must be *extremely* careful to limit its stack usage.  This
 108 +        * could include kprobes and a hypothetical future IST-less #DB
 109 +        * handler.
 110 +        */
 111 +
 112 +       cmovzq  PER_CPU_VAR(irq_stack_ptr), %rsp
 113 +       pushq   \old_rsp
 114 +.endm
 115 +
 116 +/*
 117 + * Undoes ENTER_IRQ_STACK.
 118 + */
 119 +.macro LEAVE_IRQ_STACK
 120 +       DEBUG_ENTRY_ASSERT_IRQS_OFF
 121 +       /* We need to be off the IRQ stack before decrementing irq_count. */
 122 +       popq    %rsp
 123 +
 124 +       /*
 125 +        * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
 126 +        * the irq stack but we're not on it.
 127 +        */
 128 +
 129 +       decl    PER_CPU_VAR(irq_count)
 130 +.endm
 131 +
 132  /*
 133   * Interrupt entry/exit.
 134   *
 135 @@ -485,17 +538,7 @@ END(irq_entries_start)
 136         CALL_enter_from_user_mode
 137
 138  1:
 139 -       /*
 140 -        * Save previous stack pointer, optionally switch to interrupt stack.
 141 -        * irq_count is used to check if a CPU is already on an interrupt stack
 142 -        * or not. While this is essentially redundant with preempt_count it is
 143 -        * a little cheaper to use a separate counter in the PDA (short of
 144 -        * moving irq_enter into assembly, which would be too much work)
 145 -        */
 146 -       movq    %rsp, %rdi
 147 -       incl    PER_CPU_VAR(irq_count)
 148 -       cmovzq  PER_CPU_VAR(irq_stack_ptr), %rsp
 149 -       pushq   %rdi
 150 +       ENTER_IRQ_STACK old_rsp=%rdi
 151         /* We entered an interrupt context - irqs are off: */
 152         TRACE_IRQS_OFF
 153
 154 @@ -515,10 +558,8 @@ common_interrupt:
 155  ret_from_intr:
 156         DISABLE_INTERRUPTS(CLBR_ANY)
 157         TRACE_IRQS_OFF
 158 -       decl    PER_CPU_VAR(irq_count)
 159
 160 -       /* Restore saved previous stack */
 161 -       popq    %rsp
 162 +       LEAVE_IRQ_STACK
 163
 164         testb   $3, CS(%rsp)
 165         jz      retint_kernel
 166 @@ -892,12 +933,10 @@ bad_gs:
 167  ENTRY(do_softirq_own_stack)
 168         pushq   %rbp
 169         mov     %rsp, %rbp
 170 -       incl    PER_CPU_VAR(irq_count)
 171 -       cmove   PER_CPU_VAR(irq_stack_ptr), %rsp
 172 -       push    %rbp                            /* frame pointer backlink */
 173 +       ENTER_IRQ_STACK old_rsp=%r11
 174         call    __do_softirq
 175 +       LEAVE_IRQ_STACK
 176         leaveq
 177 -       decl    PER_CPU_VAR(irq_count)
 178         ret
 179  END(do_softirq_own_stack)
 180
 181 @@ -924,13 +963,11 @@ ENTRY(xen_do_hypervisor_callback)         /* do_hypervisor_callback(struct *pt_regs) */
 182   * see the correct pointer to the pt_regs
 183   */
 184         movq    %rdi, %rsp                      /* we don't return, adjust the stack frame */
 185 -11:    incl    PER_CPU_VAR(irq_count)
 186 -       movq    %rsp, %rbp
 187 -       cmovzq  PER_CPU_VAR(irq_stack_ptr), %rsp
 188 -       pushq   %rbp                            /* frame pointer backlink */
 189 +
 190 +       ENTER_IRQ_STACK old_rsp=%r10
 191         call    xen_evtchn_do_upcall
 192 -       popq    %rsp
 193 -       decl    PER_CPU_VAR(irq_count)
 194 +       LEAVE_IRQ_STACK
 195 +
 196  #ifndef CONFIG_PREEMPT
 197         call    xen_maybe_preempt_hcall
 198  #endif
 199 --
 200 2.14.2
 201