patches/kernel/0151-x86-entry-64-Separate-cpu_current_top_of_stack-from-.patch

   1 From c2cd64d7bc24a46e3192246a97b30ca5a9692d42 Mon Sep 17 00:00:00 2001
   2 From: Andy Lutomirski <luto@kernel.org>
   3 Date: Mon, 4 Dec 2017 15:07:21 +0100
   4 Subject: [PATCH 151/241] x86/entry/64: Separate cpu_current_top_of_stack from
   5  TSS.sp0
   6 MIME-Version: 1.0
   7 Content-Type: text/plain; charset=UTF-8
   8 Content-Transfer-Encoding: 8bit
   9
  10 CVE-2017-5754
  11
  12 On 64-bit kernels, we used to assume that TSS.sp0 was the current
  13 top of stack.  With the addition of an entry trampoline, this will
  14 no longer be the case.  Store the current top of stack in TSS.sp1,
  15 which is otherwise unused but shares the same cacheline.
  16
  17 Signed-off-by: Andy Lutomirski <luto@kernel.org>
  18 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  19 Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
  20 Reviewed-by: Borislav Petkov <bp@suse.de>
  21 Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
  22 Cc: Borislav Petkov <bp@alien8.de>
  23 Cc: Borislav Petkov <bpetkov@suse.de>
  24 Cc: Brian Gerst <brgerst@gmail.com>
  25 Cc: Dave Hansen <dave.hansen@intel.com>
  26 Cc: Dave Hansen <dave.hansen@linux.intel.com>
  27 Cc: David Laight <David.Laight@aculab.com>
  28 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  29 Cc: Eduardo Valentin <eduval@amazon.com>
  30 Cc: Greg KH <gregkh@linuxfoundation.org>
  31 Cc: H. Peter Anvin <hpa@zytor.com>
  32 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
  33 Cc: Juergen Gross <jgross@suse.com>
  34 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  35 Cc: Peter Zijlstra <peterz@infradead.org>
  36 Cc: Rik van Riel <riel@redhat.com>
  37 Cc: Will Deacon <will.deacon@arm.com>
  38 Cc: aliguori@amazon.com
  39 Cc: daniel.gruss@iaik.tugraz.at
  40 Cc: hughd@google.com
  41 Cc: keescook@google.com
  42 Link: https://lkml.kernel.org/r/20171204150606.050864668@linutronix.de
  43 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  44 (cherry picked from commit 9aaefe7b59ae00605256a7d6bd1c1456432495fc)
  45 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  46 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  47 (cherry picked from commit 281be4ff07f7c67dc2a9c75ab24a7b9ff25544ae)
  48 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  49 ---
  50  arch/x86/include/asm/processor.h   | 18 +++++++++++++-----
  51  arch/x86/include/asm/thread_info.h |  2 +-
  52  arch/x86/kernel/asm-offsets_64.c   |  1 +
  53  arch/x86/kernel/process.c          | 10 ++++++++++
  54  arch/x86/kernel/process_64.c       |  1 +
  55  5 files changed, 26 insertions(+), 6 deletions(-)
  56
  57 diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
  58 index 55885465c3a7..1bfe4bad797a 100644
  59 --- a/arch/x86/include/asm/processor.h
  60 +++ b/arch/x86/include/asm/processor.h
  61 @@ -303,7 +303,13 @@ struct x86_hw_tss {
  62  struct x86_hw_tss {
  63         u32                     reserved1;
  64         u64                     sp0;
  65 +
  66 +       /*
  67 +        * We store cpu_current_top_of_stack in sp1 so it's always accessible.
  68 +        * Linux does not use ring 1, so sp1 is not otherwise needed.
  69 +        */
  70         u64                     sp1;
  71 +
  72         u64                     sp2;
  73         u64                     reserved2;
  74         u64                     ist[7];
  75 @@ -362,6 +368,8 @@ DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss);
  76
  77  #ifdef CONFIG_X86_32
  78  DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
  79 +#else
  80 +#define cpu_current_top_of_stack cpu_tss.x86_tss.sp1
  81  #endif
  82
  83  /*
  84 @@ -533,12 +541,12 @@ static inline void native_swapgs(void)
  85
  86  static inline unsigned long current_top_of_stack(void)
  87  {
  88 -#ifdef CONFIG_X86_64
  89 -       return this_cpu_read_stable(cpu_tss.x86_tss.sp0);
  90 -#else
  91 -       /* sp0 on x86_32 is special in and around vm86 mode. */
  92 +       /*
  93 +        *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
  94 +        *  and around vm86 mode and sp0 on x86_64 is special because of the
  95 +        *  entry trampoline.
  96 +        */
  97         return this_cpu_read_stable(cpu_current_top_of_stack);
  98 -#endif
  99  }
 100
 101  static inline bool on_thread_stack(void)
 102 diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
 103 index ec8ef3bbb7dc..760dd8a73927 100644
 104 --- a/arch/x86/include/asm/thread_info.h
 105 +++ b/arch/x86/include/asm/thread_info.h
 106 @@ -214,7 +214,7 @@ static inline int arch_within_stack_frames(const void * const stack,
 107  #else /* !__ASSEMBLY__ */
 108
 109  #ifdef CONFIG_X86_64
 110 -# define cpu_current_top_of_stack (cpu_tss + TSS_sp0)
 111 +# define cpu_current_top_of_stack (cpu_tss + TSS_sp1)
 112  #endif
 113
 114  #endif
 115 diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
 116 index c21a5315b38e..048f68ff3396 100644
 117 --- a/arch/x86/kernel/asm-offsets_64.c
 118 +++ b/arch/x86/kernel/asm-offsets_64.c
 119 @@ -65,6 +65,7 @@ int main(void)
 120
 121         OFFSET(TSS_ist, tss_struct, x86_tss.ist);
 122         OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
 123 +       OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
 124         BLANK();
 125
 126  #ifdef CONFIG_CC_STACKPROTECTOR
 127 diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
 128 index aa86e810fb54..407fc37a8718 100644
 129 --- a/arch/x86/kernel/process.c
 130 +++ b/arch/x86/kernel/process.c
 131 @@ -55,6 +55,16 @@ __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
 132                  * Poison it.
 133                  */
 134                 .sp0 = (1UL << (BITS_PER_LONG-1)) + 1,
 135 +
 136 +#ifdef CONFIG_X86_64
 137 +               /*
 138 +                * .sp1 is cpu_current_top_of_stack.  The init task never
 139 +                * runs user code, but cpu_current_top_of_stack should still
 140 +                * be well defined before the first context switch.
 141 +                */
 142 +               .sp1 = TOP_OF_INIT_STACK,
 143 +#endif
 144 +
 145  #ifdef CONFIG_X86_32
 146                 .ss0 = __KERNEL_DS,
 147                 .ss1 = __KERNEL_CS,
 148 diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
 149 index 01b119bebb68..157f81816915 100644
 150 --- a/arch/x86/kernel/process_64.c
 151 +++ b/arch/x86/kernel/process_64.c
 152 @@ -461,6 +461,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 153          * Switch the PDA and FPU contexts.
 154          */
 155         this_cpu_write(current_task, next_p);
 156 +       this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
 157
 158         /* Reload sp0. */
 159         update_sp0(next_p);
 160 --
 161 2.14.2
 162